pi-roundtable 0.9.7 → 0.9.9

package/src/core/attachments/tool-attachment.ts

@@ -1,6 +1,7 @@
 import { join } from "node:path";
+import { RECORDS } from "./attachment-dir.ts";
+import { expiredMessage, wasExpired } from "./attachment-expiry.ts";
 import { isAttachmentName, plainText } from "./attachment-name.ts";
-import { RECORDS } from "./attachment-store.ts";
 
 /** A file someone attached to the conversation, as a tool reads it. */
 export interface ToolAttachment {
@@ -19,6 +20,15 @@ export class AttachmentLookupError extends Error {
 	override name = "AttachmentLookupError";
 }
 
+/** A file the core removed after its retention period; the message says so, with no path. */
+export class AttachmentExpiredError extends AttachmentLookupError {
+	override name = "AttachmentExpiredError";
+
+	constructor(shown: string) {
+		super(expiredMessage(shown));
+	}
+}
+
 interface Recorded {
 	name?: unknown;
 	contentType?: unknown;
@@ -36,10 +46,12 @@ export async function openAttachment(
 	if (!isAttachmentName(file))
 		throw new AttachmentLookupError(`"${shown}" is not an attachment name`);
 	const handle = Bun.file(join(dir, file));
-	if (!(await handle.exists()))
+	if (!(await handle.exists())) {
+		if (await wasExpired(dir, file)) throw new AttachmentExpiredError(shown);
 		throw new AttachmentLookupError(
 			`no attachment named "${shown}" in this channel`,
 		);
+	}
 	const record = Bun.file(join(dir, RECORDS, `${file}.json`));
 	const recorded: Recorded = (await record.exists()) ? await record.json() : {};
 	return {

package/src/core/config/attachments.ts

@@ -0,0 +1,28 @@
+import type { AttachmentRetention } from "../attachments/attachment-retention.ts";
+import { integer, optional, shape } from "./schema.ts";
+
+/** The configuration's `attachments`: what the host does with the files people send. */
+export interface AttachmentsConfig {
+	/**
+	 * Removes the files turns used once they are `maxAgeMs` old (counted from the turn that took
+	 * the file), giving the owners' used-bytes allowance back; sweeps when the host starts and every
+	 * `sweepEveryMs` (default one hour, at most `maxAgeMs`). Left out, used files live as long as
+	 * their conversation. Files waiting for a turn are not touched; the plugin that took them
+	 * discards those.
+	 */
+	retention?: AttachmentRetention;
+}
+
+const YEAR_MS = 365 * 24 * 60 * 60_000;
+
+/** The `attachments` key's shape. */
+export const attachmentsShape = optional(
+	shape({
+		retention: optional(
+			shape({
+				maxAgeMs: integer(1, YEAR_MS),
+				sweepEveryMs: optional(integer(1, YEAR_MS)),
+			}),
+		),
+	}),
+);

package/src/core/config/config.ts

@@ -26,6 +26,7 @@ import {
 	type ResolvedDiscord,
 	resolvedDiscord,
 } from "./adapters.ts";
+import { type AttachmentsConfig, attachmentsShape } from "./attachments.ts";
 import {
 	bool,
 	guarded,
@@ -98,6 +99,8 @@ export interface RoundtableConfig {
 	database: { url: string };
 	/** Where the process keeps its files, such as pictures, attachments, and skills. */
 	dataDir: string;
+	/** What the host does with the files people send, such as removing them after a retention period. */
+	attachments?: AttachmentsConfig;
 	/** Pi's agent directory, holding the model logins; default `<dataDir>/pi`. */
 	agentDir?: string;
 	/** The model of the agents, written `<provider>/<id>`. */
@@ -211,6 +214,7 @@ const schema = shape({
 	discord: optional(discordShape),
 	database: shape({ url: text }),
 	dataDir: text,
+	attachments: attachmentsShape,
 	agentDir: optional(text),
 	model: text,
 	thinking: optional(oneOf(...THINKING_LEVELS)),
@@ -306,6 +310,8 @@ export interface ResolvedConfig {
 	discord?: ResolvedDiscord;
 	databaseUrl: string;
 	dataDir: string;
+	/** Absent `retention`: used files live as long as their conversation. */
+	attachments: AttachmentsConfig;
 	agentDir: string;
 	model: ModelRef;
 	thinking: ThinkingLevel;
@@ -421,6 +427,9 @@ export function resolveConfig(input: unknown): ResolvedConfig {
 		...(discord ? { discord: resolvedDiscord(discord, slug) } : {}),
 		databaseUrl: config.database.url,
 		dataDir: config.dataDir,
+		attachments: config.attachments?.retention
+			? { retention: config.attachments.retention }
+			: {},
 		agentDir: config.agentDir ?? `${config.dataDir}/pi`,
 		model,
 		thinking,

package/src/core/contract/attachments.ts

@@ -44,6 +44,26 @@ export interface TurnAttachmentOptions {
 	usedBytesLimit?: number;
 }
 
+/** What `expireUsed` asks: how old a used file must be to go. */
+export interface ExpireUsedOptions {
+	/** Files a turn used more than this many milliseconds ago are removed. */
+	olderThanMs: number;
+}
+
+/** What `expireUsed` removed. */
+export interface ExpiredAttachments {
+	/** How many used files were removed. */
+	files: number;
+	/** Their bytes; the owners' used-bytes tallies shrank by as much, less `unattributedBytes`. */
+	bytes: number;
+	/**
+	 * Bytes of removed files whose owner the core could not tell, because a turn used them before
+	 * the core recorded owners and their conversation has more than one person's tally. The files
+	 * are gone, but no tally shrank for them; deleting the conversation clears the tallies.
+	 */
+	unattributedBytes: number;
+}
+
 /**
  * Files a plugin takes from a person outside a turn, such as an upload over HTTP, kept for the
  * conversation until a turn uses them. A saved file waits for its person: only the principal that
@@ -89,4 +109,17 @@ export interface AttachmentPort {
 	discardPending(olderThan: Date): Promise<number>;
 	/** The bytes the principal has saved that no turn used yet, across their conversations. */
 	pendingBytes(principalId: string): Promise<number>;
+	/**
+	 * Removes the files turns used more than `olderThanMs` ago, and lowers each owner's used-bytes
+	 * tally for the conversation by exactly the bytes removed, so the allowance `usedBytesLimit`
+	 * checks comes back. The age counts from the turn that took the file, not from its upload. It
+	 * runs one after another with that person's other calls, so it never races a message taking
+	 * files. A removed file leaves a mark without its name: `read_attachment` and
+	 * `ToolTurn.attachment()` then say the file was removed after its retention period. Files
+	 * waiting for a turn are not touched; `discardPending` is theirs. A host sets
+	 * `attachments.retention` to run this on a timer. Optional, so a port a plugin wrote before it
+	 * existed still type-checks; the host's own port always has it, and a retention setting refuses
+	 * a port without it.
+	 */
+	expireUsed?(options: ExpireUsedOptions): Promise<ExpiredAttachments>;
 }

package/src/core/define-roundtable.ts

@@ -279,6 +279,9 @@ export async function defineRoundtable(
 			},
 			database: { url: config.databaseUrl },
 			dataDir: config.dataDir,
+			...(config.attachments.retention
+				? { attachments: { retention: config.attachments.retention } }
+				: {}),
 			toolTiers: toolTiers(config.toolTiers, logger),
 			listeners: [
 				...(config.http ? [publicListener(config.http)] : []),

package/src/core/define.ts

@@ -1,17 +1,14 @@
 import type { Static, TObject } from "typebox";
-import {
-	AttachmentLookupError,
-	openAttachment,
-	type ToolAttachment,
-} from "./attachments/tool-attachment.ts";
+import type { ToolAttachment } from "./attachments/tool-attachment.ts";
 import type { ReplyFile } from "./domain/conversation.ts";
 import { PluginError } from "./errors.ts";
-import type { HoldRule } from "./holds.ts";
+import { type HoldRule, isPromise } from "./holds.ts";
+import { messages } from "./i18n/index.ts";
 import { type RoundtablePlugin, refuseRemovedFields } from "./plugin.ts";
-import { attachReplyFile } from "./reply-files.ts";
 import type { AgentTurnScope, ChannelKey, SessionTool } from "./sessions.ts";
 import { toolError, toolText } from "./shared/tool-result.ts";
 import { type Speaker, TIERS, type Tier } from "./speakers.ts";
+import { toolTurn } from "./tool-turn.ts";
 
 /** Tool names are lowercase words joined by underscores, like every tool the core ships. */
 const TOOL_NAME = /^[a-z][a-z0-9]*(_[a-z0-9]+)*$/;
@@ -64,8 +61,22 @@ export interface ToolSpec<Schema extends TObject> {
 	minTier: Tier;
 	/** Whether agents carry the tool in their turns; default true. */
 	agent?: boolean;
-	/** A short description of what the call would do when it must wait for approval; undefined lets it run. */
-	hold?: (args: Static<Schema>) => string | undefined;
+	/**
+	 * A short description of what the call would do when it must wait for approval; undefined lets it
+	 * run. It may answer later, after looking something up as the turn's speaker, such as a name to
+	 * show instead of an id: `turn` is the turn `run` would get, and its `signal` aborts when the
+	 * turn stops or the wait ends. The owner's approval card waits for it up to
+	 * `HOLD_DESCRIBE_TIMEOUT_MS`; a rejection or a timeout holds the call under a generic
+	 * description, so a hold that fails never lets the call run. It must change nothing.
+	 *
+	 * A host that checks the hold with no turn, such as a precheck script's call, never calls a hold
+	 * that declares `turn`; it sees the generic description. A hold that declares only `args` and
+	 * answers at once is called there as it always was.
+	 */
+	hold?: (
+		args: Static<Schema>,
+		turn: ToolTurn,
+	) => string | undefined | Promise<string | undefined>;
 	run(args: Static<Schema>, turn: ToolTurn): Promise<string> | string;
 }
 
@@ -124,29 +135,10 @@ export function defineTool<Schema extends TObject>(
 					execute: async (_toolCallId, params, signal) => {
 						try {
 							return toolText(
-								await spec.run(params as Static<Schema>, {
-									speaker: context.speaker(),
-									channel: context.turnChannel,
-									agent: context.agent,
-									signal,
-									...(context.workspace
-										? { workspace: context.workspace }
-										: {}),
-									attachFile: attachReplyFile,
-									attachment: async (file) => {
-										if (context.attachmentDir === undefined)
-											throw new ToolRefusal(
-												"this session keeps no attachments",
-											);
-										try {
-											return await openAttachment(context.attachmentDir, file);
-										} catch (error) {
-											if (error instanceof AttachmentLookupError)
-												throw new ToolRefusal(error.message);
-											throw error;
-										}
-									},
-								}),
+								await spec.run(
+									params as Static<Schema>,
+									toolTurn(context, signal),
+								),
 							);
 						} catch (error) {
 							if (error instanceof ToolRefusal) return toolError(error.message);
@@ -170,11 +162,38 @@ export function defineTool<Schema extends TObject>(
 		hold: {
 			name: `tool:${name}`,
 			describe: (tool, input) =>
-				tool === name ? hold(input as Static<Schema>) : undefined,
+				tool === name ? holdWithoutTurn(name, hold, input) : undefined,
+			describeInTurn: (tool, input, _context, turn) =>
+				tool === name ? hold(input as Static<Schema>, turn) : undefined,
 		},
 	};
 }
 
+/**
+ * A tool's hold as a host with no turn sees it: called with the arguments alone, unless it declares
+ * the turn, and described generically when it needs the turn or answers later.
+ */
+function holdWithoutTurn<Schema extends TObject>(
+	name: string,
+	hold: NonNullable<ToolSpec<Schema>["hold"]>,
+	input: Record<string, unknown>,
+): string | undefined {
+	if (hold.length >= 2) return messages().holdGeneric(name);
+	try {
+		const answer = (
+			hold as (
+				args: Static<Schema>,
+			) => string | undefined | Promise<string | undefined>
+		)(input as Static<Schema>);
+		if (!isPromise(answer)) return answer;
+		// Its answer is not waited for here; a rejection must not surface on its own.
+		Promise.resolve(answer).catch(() => undefined);
+	} catch {
+		// A hold that needs the turn without declaring it, or that fails: it still counts as held.
+	}
+	return messages().holdGeneric(name);
+}
+
 /**
  * Defines a plugin. It returns the plugin it was given, typed, after checking its name, so a
  * mistake shows where the plugin is written.

package/src/core/holds.ts

@@ -1,6 +1,13 @@
+import type { ToolTurn } from "./define.ts";
 import { PluginError } from "./errors.ts";
 import { type Tier, tierAtLeast } from "./speakers.ts";
 
+/**
+ * How long the confirmation gate waits for a hold that looks things up before it describes the
+ * call generically and holds it anyway, in milliseconds.
+ */
+export const HOLD_DESCRIBE_TIMEOUT_MS = 10_000;
+
 /** What a hold rule knows about the session making the call. */
 export interface HoldContext {
 	/** The shared workspace of a session with a shell; calls reaching outside it may be held. */
@@ -27,6 +34,20 @@ export interface HoldRule {
 	 * `describe` with an empty input.
 	 */
 	mayHold?(tool: string): boolean;
+	/**
+	 * `describe` for a rule that must look something up as the turn's speaker before it can describe
+	 * the call, which it may answer later. The confirmation gate asks it, instead of `describe`,
+	 * when a turn makes the call, and waits for the answer up to `HOLD_DESCRIBE_TIMEOUT_MS`; a
+	 * rejection or a timeout holds the call under a generic description. `turn.signal` aborts when the
+	 * turn stops or the wait ends. `describe` stays what a host without a turn sees, such as a
+	 * precheck script's call, and answers at once.
+	 */
+	describeInTurn?(
+		tool: string,
+		input: Record<string, unknown>,
+		context: HoldContext,
+		turn: ToolTurn,
+	): string | undefined | Promise<string | undefined>;
 	/**
 	 * The lowest tier that may approve a call this rule holds, when it is higher than the tool's own:
 	 * for a call that stands for others, such as saving a script whose runs make held calls.
@@ -46,6 +67,17 @@ export type HoldCheck = ((
 ) => string | undefined) & {
 	/** The name of a rule that may hold some call of `tool` whatever its input; see `HoldRule.mayHold`. */
 	mayHold?(tool: string): string | undefined;
+	/**
+	 * The check as a turn runs it: each rule's `describeInTurn`, or its `describe` for a rule
+	 * without one, asked in order until one describes the call. It answers at once when every rule
+	 * asked did, and otherwise as a promise that rejects when a rule's does.
+	 */
+	inTurn?(
+		tool: string,
+		input: Record<string, unknown>,
+		context: HoldContext,
+		turn: ToolTurn,
+	): string | undefined | Promise<string | undefined>;
 	/** The highest tier any rule requires to approve the call; see `HoldRule.approvalTier`. */
 	approvalTier?(
 		tool: string,
@@ -81,6 +113,26 @@ export function holdChain(rules: readonly HoldRule[]): HoldCheck {
 		}
 		return undefined;
 	};
+	check.inTurn = (tool, input, context, turn) => {
+		const ask = (
+			from: number,
+		): string | undefined | Promise<string | undefined> => {
+			for (let at = from; at < rules.length; at++) {
+				const rule = rules[at];
+				if (!rule) continue;
+				const description = rule.describeInTurn
+					? rule.describeInTurn(tool, input, context, turn)
+					: rule.describe(tool, input, context);
+				if (isPromise(description))
+					return description.then((found) =>
+						found !== undefined ? found : ask(at + 1),
+					);
+				if (description !== undefined) return description;
+			}
+			return undefined;
+		};
+		return ask(0);
+	};
 	check.mayHold = (tool) => rules.find((rule) => rule.mayHold?.(tool))?.name;
 	check.approvalTier = (tool, input, context) =>
 		rules.reduce<Tier | undefined>(
@@ -90,3 +142,12 @@ export function holdChain(rules: readonly HoldRule[]): HoldCheck {
 		);
 	return check;
 }
+
+/** Whether a hold's answer comes later. */
+export function isPromise<T>(value: T | Promise<T>): value is Promise<T> {
+	return (
+		typeof value === "object" &&
+		value !== null &&
+		typeof (value as { then?: unknown }).then === "function"
+	);
+}

package/src/core/host.ts

@@ -1,5 +1,9 @@
 import type { SQL } from "bun";
 import { attachmentPort } from "./attachments/attachment-port.ts";
+import {
+	type AttachmentRetention,
+	retentionServices,
+} from "./attachments/attachment-retention.ts";
 import type { ConversationPort } from "./contract/channels.ts";
 import type { SurfacePort } from "./contract/surface.ts";
 import { openPool, runMigrations } from "./db/migrations.ts";
@@ -82,6 +86,13 @@ export interface RoundtableOptions {
 	database?: { url: string };
 	/** The directory the host keeps files in; `context.attachments` keeps its files there. Without it that port refuses every call. */
 	dataDir?: string;
+	/**
+	 * What the host does with the attachments it keeps. `retention` removes the files turns used once
+	 * they are `maxAgeMs` old, sweeping at start and every `sweepEveryMs` (default one hour), and gives
+	 * the owners' used-bytes allowance back; without it used files live as long as their conversation.
+	 * Needs `dataDir`.
+	 */
+	attachments?: { retention?: AttachmentRetention };
 	/** Receives the work a shutdown drain gave up on, before any service stops. */
 	aborted?: (left: string[]) => Promise<void>;
 	/**
@@ -270,6 +281,17 @@ export class Roundtable {
 		this.#services = new ServiceRegistry(this.#active);
 		this.#services.checkRequires();
 		const providers = resolveProviders(this.#active, this.#options.judgeModel);
+		const attachments = attachmentPort({
+			dataDir: this.#options.dataDir,
+			registry: () => this.#services?.find(CONVERSATIONS),
+			logger,
+		});
+		const retention = retentionServices(
+			this.#options.attachments?.retention,
+			this.#options.dataDir,
+			attachments,
+			logger,
+		);
 		await this.#migrate();
 		this.#registry = await collectContributions(
 			this.#active,
@@ -287,11 +309,7 @@ export class Roundtable {
 				toolTiers: this.#tiers,
 				events: this.#events.sink,
 				conversations: this.#conversations(),
-				attachments: attachmentPort({
-					dataDir: this.#options.dataDir,
-					registry: () => this.#services?.find(CONVERSATIONS),
-					logger,
-				}),
+				attachments,
 				surfaces: this.#surfaces(),
 				turns: this.#turns(),
 				database: () => {
@@ -331,7 +349,7 @@ export class Roundtable {
 		});
 		const http = new HttpListeners(listeners, routes, logger);
 		for (const plugin of this.#active) await plugin.preflight?.();
-		for (const service of this.#registry.services) {
+		for (const service of [...this.#registry.services, ...retention]) {
 			await service.start?.();
 			this.#started.push(service);
 		}

package/src/core/i18n/conversation.ts

@@ -28,6 +28,7 @@ export function conversationEn(ctx: CatalogContext) {
 		answerReturned: (text: string) => `↩️ Reply:\n${text}`,
 		thinkingAuto: "auto",
 		confirmTitle: (asker: string) => `${asker} wants to run this action`,
+		holdGeneric: (tool: string) => `run ${tool}`,
 		askTitle: (asker: string) => `${asker} has a question for you`,
 		cardFile: (path: string, bytes: number | undefined) =>
 			bytes === undefined
@@ -59,6 +60,7 @@ export function conversationZhTW(
 		answerReturned: (text: string) => `↩️ 回覆：\n${text}`,
 		thinkingAuto: "自動",
 		confirmTitle: (asker: string) => `${asker} 要執行這個動作`,
+		holdGeneric: (tool: string) => `執行 ${tool}`,
 		askTitle: (asker: string) => `${asker} 想問你`,
 		cardFile: (path: string, bytes: number | undefined) =>
 			bytes === undefined