pi-roundtable 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/docs/plugins.md +5 -2
- package/package.json +1 -1
- package/src/core/attachments/attachment-dir.ts +51 -0
- package/src/core/attachments/attachment-fetcher.ts +30 -6
- package/src/core/attachments/attachment-name.ts +34 -0
- package/src/core/attachments/attachment-store.ts +169 -54
- package/src/core/attachments/prompt-block.ts +6 -2
- package/src/core/attachments/tool-attachment.ts +14 -5
- package/src/core/contract/attachments.ts +23 -2
- package/src/core/runtime/pi-agent-runtime.ts +5 -1
- package/src/core/shared/attachment-reader.ts +11 -4
- package/src/index.ts +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,25 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [0.9.3] - 2026-10-10
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- A long non-ASCII upload name no longer fails with `ENAMETOOLONG` on Linux: a stored file name is cut by UTF-8 bytes (150 at most, on a code point, keeping the end so the extension stays) instead of by UTF-16 units, so the file name and its `.json` record fit the 255 bytes a file system takes. A leading dot left by the cut is replaced too. The Discord download path shares the fix. `AttachmentPort.save` removes the data file when the record write fails, so a half-saved file no longer counts as waiting.
|
|
13
|
+
- Calls on one principal's attachments run one after another, so two messages that name overlapping files no longer leave the refused one's other files moved into the conversation: `turnAttachments` moves the whole set or none, rolls back a partial move, and answers a file a sweep or a message took meanwhile with `AttachmentRefusal` (`unknown_file`) instead of a raw `ENOENT`.
|
|
14
|
+
- The `## Attachments` block prints a name and a content type a prompt can trust: control characters, line and paragraph separators and bidi controls in a name become underscores and quotes become apostrophes, and a content type that is not a plain `type/subtype` prints as `application/octet-stream`.
|
|
15
|
+
- `ToolTurn.attachment()` and `read_attachment` refuse a name with a NUL byte as "not an attachment name"; a NUL name no longer surfaces a runtime error that quoted the host's attachment directory, and a file that cannot be read reports its name, not its path.
|
|
16
|
+
- `discardPending` removes the staging directories it emptied, so a conversation's uploads leave no empty directories behind.
|
|
17
|
+
- `deleteConversation` of the Pi runtime removes the conversation's attachment directory, the files saved for it that no turn used, and its share of the used-bytes tally, so a deleted conversation keeps no files (a conversation deleted in the instant a message is taking its files can leave that message's share of the tally behind).
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
|
|
21
|
+
- `TurnAttachmentOptions` and `AttachmentPort.turnAttachments(channel, principalId, files, options?)`: `options.usedBytesLimit` caps the bytes one principal's turns keep across all their conversations. A call that would pass it moves nothing and is refused with the new `AttachmentRefusalCode` `quota_exceeded`; without it the core sets no limit. The core tallies the bytes per principal and conversation, and deleting a conversation gives its bytes back.
|
|
22
|
+
|
|
23
|
+
### Documented
|
|
24
|
+
|
|
25
|
+
- The guide says what the port checks (a private conversation's record) and what it does not: a channel with no record, or any channel on a host with no conversation registry, is admitted for every principal, so the plugin that owns a channel checks that the caller may use it before every call.
|
|
26
|
+
|
|
8
27
|
## [0.9.2] - 2026-10-10
|
|
9
28
|
|
|
10
29
|
### Added
|
package/docs/plugins.md
CHANGED
|
@@ -538,14 +538,16 @@ A plugin whose clients upload a file themselves, over HTTP for instance, has the
|
|
|
538
538
|
| Call | What it does |
|
|
539
539
|
|---|---|
|
|
540
540
|
| `save(channel, principalId, { name, contentType, data })` | Keeps the file for that principal in that conversation and returns a `StoredAttachment`; its `file` is the name to hand back to the client. Refuses a file over 25 MiB (`too_large`) |
|
|
541
|
-
| `turnAttachments(channel, principalId, files)` | Takes the `file` names `save` returned and returns the turn's `TurnAttachments`; pass it as `attachments` to [`turns.run`](#personas-and-contextturns-conversations-of-a-kind-of-your-own). Each file moves into the conversation's attachment directory, up to four images are prepared for the model, and the turn's prompt lists every file under `## Attachments`. Refuses the whole set when one name is unknown or already used (`unknown_file`); then none moves |
|
|
541
|
+
| `turnAttachments(channel, principalId, files, options?)` | Takes the `file` names `save` returned and returns the turn's `TurnAttachments`; pass it as `attachments` to [`turns.run`](#personas-and-contextturns-conversations-of-a-kind-of-your-own). Each file moves into the conversation's attachment directory, up to four images are prepared for the model, and the turn's prompt lists every file under `## Attachments`. Refuses the whole set when one name is unknown or already used (`unknown_file`), or when `options.usedBytesLimit` would be passed (`quota_exceeded`); then none moves, even when other calls for the same principal run at once |
|
|
542
542
|
| `remove(channel, principalId, file)` | Discards a saved file no turn used; `false` when there is none |
|
|
543
543
|
| `discardPending(olderThan)` | Discards every saved file no turn used that was saved before the date, and returns how many; call it from a service's timer so abandoned uploads do not pile up |
|
|
544
544
|
| `pendingBytes(principalId)` | The bytes the principal has saved that no turn used yet, for a quota of your own |
|
|
545
545
|
|
|
546
546
|
A saved file waits for its owner: only the principal that saved it can use it, and only in the conversation it was saved for.
|
|
547
547
|
The conversation's record decides who may save into it: a private conversation refuses every principal but its own with an `AttachmentRefusal` whose `code` is `forbidden`, and a shared one lets each principal save and use their own files.
|
|
548
|
-
|
|
548
|
+
**The plugin that owns a channel checks that the caller may use it, before every call.** The port knows only the conversation's record: a channel with no record yet, such as a conversation opened but not written in, is admitted for every principal, and so is every channel on a host with no conversation registry. A plugin that takes the channel key from a person (a URL, a frame) and passes it on unchecked lets that person stage files into, and move files into the attachment directory of, a conversation that is not theirs. Check first that the channel is the person's, as the web chat does before it saves or uses a file.
|
|
549
|
+
`usedBytesLimit` caps what one principal's turns keep, across all their conversations; the core sets no limit of its own, and a deleted conversation gives its bytes back.
|
|
550
|
+
Deleting a conversation (`runtime.deleteConversation`) also removes its attachment directory, the files saved for it that no turn used, and its share of the tally.
|
|
549
551
|
The calls reject with `AttachmentRefusal`, and throw a `PluginError` when the host has no `dataDir` (a host built by `defineRoundtable` always has one).
|
|
550
552
|
The core limits only the size; the types a plugin accepts, the number of files, and the rate are its own to limit.
|
|
551
553
|
|
|
@@ -3429,6 +3431,7 @@ Import from the entries listed below; source area files are internal.
|
|
|
3429
3431
|
| `ToolTurn` | `pi-roundtable` | type |
|
|
3430
3432
|
| `TranscriptEntry` | `pi-roundtable` | type |
|
|
3431
3433
|
| `TransientTask` | `pi-roundtable` | type |
|
|
3434
|
+
| `TurnAttachmentOptions` | `pi-roundtable` | type |
|
|
3432
3435
|
| `TurnAttachments` | `pi-roundtable` | type |
|
|
3433
3436
|
| `TurnConversation` | `pi-roundtable` | type |
|
|
3434
3437
|
| `TurnEndEvent` | `pi-roundtable` | type |
|
package/package.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
|
+
import { readdir, rm, rmdir } from "node:fs/promises";
|
|
1
2
|
import { join } from "node:path";
|
|
2
3
|
import type { ChannelKey } from "../domain/conversation.ts";
|
|
3
4
|
|
|
5
|
+
/** Where saved files wait for a turn, beside the conversations' attachment directories. */
|
|
6
|
+
export const STAGING = "attachments-pending";
|
|
7
|
+
/** Where each principal's tally of used bytes per conversation is kept. */
|
|
8
|
+
export const USAGE = "attachments-used";
|
|
9
|
+
|
|
4
10
|
/** A channel key as a single path segment. */
|
|
5
11
|
export function channelSegment(channel: ChannelKey): string {
|
|
6
12
|
return channel.replace(/[^A-Za-z0-9_-]/g, "_");
|
|
@@ -13,3 +19,48 @@ export function ownerAttachmentDir(
|
|
|
13
19
|
): string {
|
|
14
20
|
return join(dataDir, "attachments", channelSegment(channel));
|
|
15
21
|
}
|
|
22
|
+
|
|
23
|
+
/** The entries of `dir`; none when it is not there. */
|
|
24
|
+
export async function listDir(dir: string): Promise<string[]> {
|
|
25
|
+
try {
|
|
26
|
+
return await readdir(dir);
|
|
27
|
+
} catch (error) {
|
|
28
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
29
|
+
throw error;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Removes `dir` when it is empty; a directory that has entries, or is gone, stays as it is. */
|
|
34
|
+
export async function removeIfEmpty(dir: string): Promise<void> {
|
|
35
|
+
try {
|
|
36
|
+
await rmdir(dir);
|
|
37
|
+
} catch (error) {
|
|
38
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
39
|
+
if (code !== "ENOTEMPTY" && code !== "EEXIST" && code !== "ENOENT")
|
|
40
|
+
throw error;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Removes everything the core keeps for a deleted conversation: the files its turns used, the
|
|
46
|
+
* files people saved for it and no turn used, and the tally of what each principal used in it.
|
|
47
|
+
*/
|
|
48
|
+
export async function discardConversationFiles(
|
|
49
|
+
dataDir: string,
|
|
50
|
+
channel: ChannelKey,
|
|
51
|
+
): Promise<void> {
|
|
52
|
+
const segment = channelSegment(channel);
|
|
53
|
+
await rm(ownerAttachmentDir(dataDir, channel), {
|
|
54
|
+
recursive: true,
|
|
55
|
+
force: true,
|
|
56
|
+
});
|
|
57
|
+
for (const area of [STAGING, USAGE]) {
|
|
58
|
+
const root = join(dataDir, area);
|
|
59
|
+
for (const principal of await listDir(root)) {
|
|
60
|
+
const dir = join(root, principal);
|
|
61
|
+
await rm(join(dir, segment), { recursive: true, force: true });
|
|
62
|
+
await rm(join(dir, `${segment}.tmp`), { force: true });
|
|
63
|
+
await removeIfEmpty(dir);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
AttachmentRef,
|
|
6
6
|
StoredAttachment,
|
|
7
7
|
} from "../domain/attachment.ts";
|
|
8
|
+
import { plainText } from "./attachment-name.ts";
|
|
8
9
|
|
|
9
10
|
export const MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024;
|
|
10
11
|
|
|
@@ -22,13 +23,36 @@ export interface FetchResult {
|
|
|
22
23
|
failures: AttachmentFailure[];
|
|
23
24
|
}
|
|
24
25
|
|
|
25
|
-
/**
|
|
26
|
+
/** The most UTF-8 bytes a stored name keeps; a file system takes 255 bytes for the whole file name. */
|
|
27
|
+
const MAX_NAME_BYTES = 150;
|
|
28
|
+
|
|
29
|
+
const utf8Length = (codePoint: number): number =>
|
|
30
|
+
codePoint < 0x80 ? 1 : codePoint < 0x800 ? 2 : codePoint < 0x10000 ? 3 : 4;
|
|
31
|
+
|
|
32
|
+
/** The end of `text` that fits `max` UTF-8 bytes, cut on a code point. */
|
|
33
|
+
function lastBytes(text: string, max: number): string {
|
|
34
|
+
const points = [...text];
|
|
35
|
+
let bytes = 0;
|
|
36
|
+
let start = points.length;
|
|
37
|
+
while (start > 0) {
|
|
38
|
+
const size = utf8Length(points[start - 1]?.codePointAt(0) ?? 0);
|
|
39
|
+
if (bytes + size > max) break;
|
|
40
|
+
bytes += size;
|
|
41
|
+
start -= 1;
|
|
42
|
+
}
|
|
43
|
+
return points.slice(start).join("");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Keeps a file name readable while removing path separators and control characters, and keeps the
|
|
48
|
+
* end of a long name (its extension) within 150 UTF-8 bytes so a prefix and a record suffix still
|
|
49
|
+
* fit the 255 bytes a file system takes.
|
|
50
|
+
*/
|
|
26
51
|
export function safeFileName(name: string): string {
|
|
27
|
-
const cleaned =
|
|
28
|
-
.normalize("NFC")
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
.slice(-120);
|
|
52
|
+
const cleaned = lastBytes(
|
|
53
|
+
plainText(name.toWellFormed().normalize("NFC").replace(/[/\\]/g, "_")),
|
|
54
|
+
MAX_NAME_BYTES,
|
|
55
|
+
).replace(/^\.+/, "_");
|
|
32
56
|
return cleaned || "file";
|
|
33
57
|
}
|
|
34
58
|
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { basename } from "node:path";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Characters that must not reach a prompt or a file name as they are: controls, the line and
|
|
5
|
+
* paragraph separators, and the bidi controls that reorder or hide the text around them.
|
|
6
|
+
*/
|
|
7
|
+
const UNSAFE_CHARS =
|
|
8
|
+
/[\p{Cc}\p{Zl}\p{Zp}\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]/gu;
|
|
9
|
+
|
|
10
|
+
/** `value` with each unsafe character replaced by an underscore. */
|
|
11
|
+
export function plainText(value: string): string {
|
|
12
|
+
return value.replace(UNSAFE_CHARS, "_");
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Whether `file` could be a name the attachment stores give out: one path segment, not hidden, no NUL. */
|
|
16
|
+
export function isAttachmentName(file: string): boolean {
|
|
17
|
+
return (
|
|
18
|
+
file !== "" &&
|
|
19
|
+
basename(file) === file &&
|
|
20
|
+
!file.startsWith(".") &&
|
|
21
|
+
!file.includes("\0")
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const TYPE_GRAMMAR = /^[a-z0-9.+-]+\/[a-z0-9.+-]+$/i;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The type a prompt may print: `type` without its parameters (`text/plain; charset=utf-8` is
|
|
29
|
+
* `text/plain`) when that is a plain type/subtype, else the generic binary type.
|
|
30
|
+
*/
|
|
31
|
+
export function plainContentType(type: string): string {
|
|
32
|
+
const base = type.split(";")[0]?.trim().toLowerCase() ?? "";
|
|
33
|
+
return TYPE_GRAMMAR.test(base) ? base : "application/octet-stream";
|
|
34
|
+
}
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { mkdirSync } from "node:fs";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
5
|
import {
|
|
6
6
|
type AttachmentPort,
|
|
7
7
|
AttachmentRefusal,
|
|
8
8
|
type AttachmentUpload,
|
|
9
|
+
type TurnAttachmentOptions,
|
|
9
10
|
} from "../contract/attachments.ts";
|
|
10
11
|
import type { ConversationRegistry } from "../conversations/conversation-registry.ts";
|
|
11
12
|
import type {
|
|
@@ -14,12 +15,18 @@ import type {
|
|
|
14
15
|
} from "../domain/attachment.ts";
|
|
15
16
|
import type { Logger } from "../log.ts";
|
|
16
17
|
import type { ChannelKey } from "../sessions.ts";
|
|
17
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
channelSegment,
|
|
20
|
+
listDir,
|
|
21
|
+
ownerAttachmentDir,
|
|
22
|
+
removeIfEmpty,
|
|
23
|
+
STAGING,
|
|
24
|
+
USAGE,
|
|
25
|
+
} from "./attachment-dir.ts";
|
|
18
26
|
import { MAX_ATTACHMENT_BYTES, safeFileName } from "./attachment-fetcher.ts";
|
|
27
|
+
import { isAttachmentName } from "./attachment-name.ts";
|
|
19
28
|
import { modelImagesOf } from "./model-images.ts";
|
|
20
29
|
|
|
21
|
-
/** Where saved files wait, beside the conversations' attachment directories. */
|
|
22
|
-
const STAGING = "attachments-pending";
|
|
23
30
|
/** The record of a saved file, in a directory next to it that no file name can be. */
|
|
24
31
|
export const RECORDS = ".records";
|
|
25
32
|
|
|
@@ -46,23 +53,28 @@ function principalSegment(principalId: string): string {
|
|
|
46
53
|
return createHash("sha256").update(principalId).digest("hex").slice(0, 32);
|
|
47
54
|
}
|
|
48
55
|
|
|
49
|
-
/**
|
|
50
|
-
function
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
56
|
+
/** The bytes of a tally file; none when it is missing or unreadable as a number. */
|
|
57
|
+
async function readTally(path: string): Promise<number> {
|
|
58
|
+
try {
|
|
59
|
+
const value = Number(await readFile(path, "utf8"));
|
|
60
|
+
return Number.isFinite(value) && value > 0 ? value : 0;
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return 0;
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
57
65
|
}
|
|
58
66
|
|
|
59
|
-
|
|
67
|
+
/** Writes a tally whole or not at all, creating its directory if a conversation's deletion removed it meanwhile. */
|
|
68
|
+
async function writeTally(path: string, bytes: number): Promise<void> {
|
|
69
|
+
const temporary = `${path}.tmp`;
|
|
60
70
|
try {
|
|
61
|
-
|
|
71
|
+
await writeFile(temporary, String(bytes));
|
|
62
72
|
} catch (error) {
|
|
63
|
-
if ((error as NodeJS.ErrnoException).code
|
|
64
|
-
|
|
73
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
74
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
75
|
+
await writeFile(temporary, String(bytes));
|
|
65
76
|
}
|
|
77
|
+
await rename(temporary, path);
|
|
66
78
|
}
|
|
67
79
|
|
|
68
80
|
/**
|
|
@@ -70,14 +82,34 @@ async function listDir(dir: string): Promise<string[]> {
|
|
|
70
82
|
* its conversation until a turn uses it, then moved into the conversation's attachment directory,
|
|
71
83
|
* the one `read_attachment` reads. The conversation's record decides who may stage: a private
|
|
72
84
|
* conversation takes only its principal's files.
|
|
85
|
+
*
|
|
86
|
+
* Calls that change one principal's files run one after another, so two messages of one person
|
|
87
|
+
* never see each other half done; the tally of the bytes a principal used per conversation lives
|
|
88
|
+
* beside the staging directories.
|
|
73
89
|
*/
|
|
74
90
|
export class AttachmentStore implements AttachmentPort {
|
|
75
91
|
readonly #options: AttachmentStoreOptions;
|
|
92
|
+
/** The last call queued for each principal, which the next one waits for. */
|
|
93
|
+
readonly #tails = new Map<string, Promise<void>>();
|
|
76
94
|
|
|
77
95
|
constructor(options: AttachmentStoreOptions) {
|
|
78
96
|
this.#options = options;
|
|
79
97
|
}
|
|
80
98
|
|
|
99
|
+
/** Runs `run` after the principal's earlier calls, whether they succeeded or not. */
|
|
100
|
+
#exclusive<T>(principal: string, run: () => Promise<T>): Promise<T> {
|
|
101
|
+
const result = (this.#tails.get(principal) ?? Promise.resolve()).then(run);
|
|
102
|
+
const tail = result.then(
|
|
103
|
+
() => undefined,
|
|
104
|
+
() => undefined,
|
|
105
|
+
);
|
|
106
|
+
this.#tails.set(principal, tail);
|
|
107
|
+
void tail.then(() => {
|
|
108
|
+
if (this.#tails.get(principal) === tail) this.#tails.delete(principal);
|
|
109
|
+
});
|
|
110
|
+
return result;
|
|
111
|
+
}
|
|
112
|
+
|
|
81
113
|
#stagingDir(channel: ChannelKey, principalId: string): string {
|
|
82
114
|
return join(
|
|
83
115
|
this.#options.dataDir,
|
|
@@ -87,6 +119,10 @@ export class AttachmentStore implements AttachmentPort {
|
|
|
87
119
|
);
|
|
88
120
|
}
|
|
89
121
|
|
|
122
|
+
#tallyDir(principalId: string): string {
|
|
123
|
+
return join(this.#options.dataDir, USAGE, principalSegment(principalId));
|
|
124
|
+
}
|
|
125
|
+
|
|
90
126
|
/** Refuses a principal other than the one a private conversation belongs to. */
|
|
91
127
|
async #admit(channel: ChannelKey, principalId: string): Promise<void> {
|
|
92
128
|
const record = await this.#options.registry()?.get(channel);
|
|
@@ -110,15 +146,25 @@ export class AttachmentStore implements AttachmentPort {
|
|
|
110
146
|
`a file may be at most 25 MiB; "${upload.name}" is ${size} bytes`,
|
|
111
147
|
);
|
|
112
148
|
const dir = this.#stagingDir(channel, principalId);
|
|
113
|
-
mkdirSync(join(dir, RECORDS), { recursive: true, mode: 0o700 });
|
|
114
149
|
const file = `${crypto.randomUUID()}-${safeFileName(upload.name)}`;
|
|
115
150
|
const path = join(dir, file);
|
|
116
|
-
|
|
151
|
+
const recordPath = join(dir, RECORDS, `${file}.json`);
|
|
117
152
|
const record: StagedRecord = {
|
|
118
153
|
name: upload.name,
|
|
119
154
|
contentType: upload.contentType,
|
|
120
155
|
};
|
|
121
|
-
await
|
|
156
|
+
await this.#exclusive(principalSegment(principalId), async () => {
|
|
157
|
+
mkdirSync(join(dir, RECORDS), { recursive: true, mode: 0o700 });
|
|
158
|
+
try {
|
|
159
|
+
await Bun.write(path, upload.data);
|
|
160
|
+
await Bun.write(recordPath, JSON.stringify(record));
|
|
161
|
+
} catch (error) {
|
|
162
|
+
// A file without its record could never be used or removed, yet would count as waiting.
|
|
163
|
+
await rm(path, { force: true });
|
|
164
|
+
await rm(recordPath, { force: true });
|
|
165
|
+
throw error;
|
|
166
|
+
}
|
|
167
|
+
});
|
|
122
168
|
return {
|
|
123
169
|
name: upload.name,
|
|
124
170
|
file,
|
|
@@ -133,8 +179,26 @@ export class AttachmentStore implements AttachmentPort {
|
|
|
133
179
|
channel: ChannelKey,
|
|
134
180
|
principalId: string,
|
|
135
181
|
files: readonly string[],
|
|
182
|
+
options: TurnAttachmentOptions = {},
|
|
136
183
|
): Promise<TurnAttachments> {
|
|
137
184
|
await this.#admit(channel, principalId);
|
|
185
|
+
const found = await this.#exclusive(principalSegment(principalId), () =>
|
|
186
|
+
this.#claim(channel, principalId, files, options),
|
|
187
|
+
);
|
|
188
|
+
const { images, failures } = await modelImagesOf(
|
|
189
|
+
found,
|
|
190
|
+
this.#options.logger,
|
|
191
|
+
);
|
|
192
|
+
return { files: found, images, failures };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Checks every file, then moves them all into the conversation, or none when any step fails. */
|
|
196
|
+
async #claim(
|
|
197
|
+
channel: ChannelKey,
|
|
198
|
+
principalId: string,
|
|
199
|
+
files: readonly string[],
|
|
200
|
+
options: TurnAttachmentOptions,
|
|
201
|
+
): Promise<StoredAttachment[]> {
|
|
138
202
|
const staging = this.#stagingDir(channel, principalId);
|
|
139
203
|
const target = ownerAttachmentDir(this.#options.dataDir, channel);
|
|
140
204
|
const found: StoredAttachment[] = [];
|
|
@@ -145,39 +209,70 @@ export class AttachmentStore implements AttachmentPort {
|
|
|
145
209
|
"unknown_file",
|
|
146
210
|
`no saved file "${file}" is waiting for this person in ${channel}`,
|
|
147
211
|
);
|
|
148
|
-
const from = join(staging, file);
|
|
149
212
|
found.push({
|
|
150
213
|
name: record.name,
|
|
151
214
|
file,
|
|
152
215
|
path: join(target, file),
|
|
153
216
|
contentType: record.contentType,
|
|
154
|
-
size: (await stat(
|
|
217
|
+
size: (await stat(join(staging, file))).size,
|
|
155
218
|
fromReference: false,
|
|
156
219
|
});
|
|
157
220
|
}
|
|
221
|
+
const added = found.reduce((sum, stored) => sum + stored.size, 0);
|
|
222
|
+
const tally = join(this.#tallyDir(principalId), channelSegment(channel));
|
|
223
|
+
const before = await readTally(tally);
|
|
224
|
+
if (options.usedBytesLimit !== undefined) {
|
|
225
|
+
const used = await this.#usedBytes(principalId);
|
|
226
|
+
if (used + added > options.usedBytesLimit)
|
|
227
|
+
throw new AttachmentRefusal(
|
|
228
|
+
"quota_exceeded",
|
|
229
|
+
`the files already used in turns hold ${used} bytes, and ${added} more would pass the limit of ${options.usedBytesLimit} bytes`,
|
|
230
|
+
);
|
|
231
|
+
}
|
|
158
232
|
mkdirSync(join(target, RECORDS), { recursive: true, mode: 0o700 });
|
|
159
|
-
|
|
160
|
-
|
|
233
|
+
mkdirSync(this.#tallyDir(principalId), { recursive: true, mode: 0o700 });
|
|
234
|
+
const moved: StoredAttachment[] = [];
|
|
235
|
+
try {
|
|
236
|
+
for (const stored of found) {
|
|
161
237
|
await rename(join(staging, stored.file), stored.path);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
`the saved file "${stored.file}" was used already`,
|
|
168
|
-
);
|
|
169
|
-
throw error;
|
|
238
|
+
moved.push(stored);
|
|
239
|
+
await rename(
|
|
240
|
+
join(staging, RECORDS, `${stored.file}.json`),
|
|
241
|
+
join(target, RECORDS, `${stored.file}.json`),
|
|
242
|
+
);
|
|
170
243
|
}
|
|
244
|
+
await writeTally(tally, before + added);
|
|
245
|
+
} catch (error) {
|
|
246
|
+
await this.#putBack(moved, staging, target, tally, before);
|
|
247
|
+
// Something else took the file between the check and the move.
|
|
248
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT")
|
|
249
|
+
throw new AttachmentRefusal(
|
|
250
|
+
"unknown_file",
|
|
251
|
+
"a saved file was used or discarded while the message was taken",
|
|
252
|
+
);
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
return found;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Undoes a partial move: the files return to staging and the tally to what it was. */
|
|
259
|
+
async #putBack(
|
|
260
|
+
moved: readonly StoredAttachment[],
|
|
261
|
+
staging: string,
|
|
262
|
+
target: string,
|
|
263
|
+
tally: string,
|
|
264
|
+
before: number,
|
|
265
|
+
): Promise<void> {
|
|
266
|
+
for (const stored of moved) {
|
|
267
|
+
await rename(stored.path, join(staging, stored.file)).catch(
|
|
268
|
+
() => undefined,
|
|
269
|
+
);
|
|
171
270
|
await rename(
|
|
172
|
-
join(staging, RECORDS, `${stored.file}.json`),
|
|
173
271
|
join(target, RECORDS, `${stored.file}.json`),
|
|
174
|
-
|
|
272
|
+
join(staging, RECORDS, `${stored.file}.json`),
|
|
273
|
+
).catch(() => undefined);
|
|
175
274
|
}
|
|
176
|
-
|
|
177
|
-
found,
|
|
178
|
-
this.#options.logger,
|
|
179
|
-
);
|
|
180
|
-
return { files: found, images, failures };
|
|
275
|
+
await writeTally(tally, before).catch(() => undefined);
|
|
181
276
|
}
|
|
182
277
|
|
|
183
278
|
async remove(
|
|
@@ -187,27 +282,37 @@ export class AttachmentStore implements AttachmentPort {
|
|
|
187
282
|
): Promise<boolean> {
|
|
188
283
|
await this.#admit(channel, principalId);
|
|
189
284
|
const staging = this.#stagingDir(channel, principalId);
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
285
|
+
return this.#exclusive(principalSegment(principalId), async () => {
|
|
286
|
+
if (!(await this.#recordOf(staging, file))) return false;
|
|
287
|
+
await rm(join(staging, file), { force: true });
|
|
288
|
+
await rm(join(staging, RECORDS, `${file}.json`), { force: true });
|
|
289
|
+
return true;
|
|
290
|
+
});
|
|
194
291
|
}
|
|
195
292
|
|
|
196
293
|
async discardPending(olderThan: Date): Promise<number> {
|
|
197
294
|
const root = join(this.#options.dataDir, STAGING);
|
|
198
295
|
let discarded = 0;
|
|
199
296
|
for (const principal of await listDir(root)) {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
for (const
|
|
203
|
-
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
297
|
+
discarded += await this.#exclusive(principal, async () => {
|
|
298
|
+
let count = 0;
|
|
299
|
+
for (const channel of await listDir(join(root, principal))) {
|
|
300
|
+
const dir = join(root, principal, channel);
|
|
301
|
+
for (const entry of await listDir(dir)) {
|
|
302
|
+
if (entry === RECORDS) continue;
|
|
303
|
+
const info = await stat(join(dir, entry)).catch(() => undefined);
|
|
304
|
+
if (!info || info.mtimeMs >= olderThan.getTime()) continue;
|
|
305
|
+
await rm(join(dir, entry), { force: true });
|
|
306
|
+
await rm(join(dir, RECORDS, `${entry}.json`), { force: true });
|
|
307
|
+
count += 1;
|
|
308
|
+
}
|
|
309
|
+
// A conversation with nothing waiting keeps no directory.
|
|
310
|
+
await removeIfEmpty(join(dir, RECORDS));
|
|
311
|
+
await removeIfEmpty(dir);
|
|
209
312
|
}
|
|
210
|
-
|
|
313
|
+
await removeIfEmpty(join(root, principal));
|
|
314
|
+
return count;
|
|
315
|
+
});
|
|
211
316
|
}
|
|
212
317
|
return discarded;
|
|
213
318
|
}
|
|
@@ -230,12 +335,22 @@ export class AttachmentStore implements AttachmentPort {
|
|
|
230
335
|
return total;
|
|
231
336
|
}
|
|
232
337
|
|
|
338
|
+
/** The bytes the principal's turns used, across their conversations. */
|
|
339
|
+
async #usedBytes(principalId: string): Promise<number> {
|
|
340
|
+
const dir = this.#tallyDir(principalId);
|
|
341
|
+
let total = 0;
|
|
342
|
+
for (const channel of await listDir(dir))
|
|
343
|
+
if (!channel.endsWith(".tmp"))
|
|
344
|
+
total += await readTally(join(dir, channel));
|
|
345
|
+
return total;
|
|
346
|
+
}
|
|
347
|
+
|
|
233
348
|
/** The record of a staged file, or undefined when it is not staged here. */
|
|
234
349
|
async #recordOf(
|
|
235
350
|
staging: string,
|
|
236
351
|
file: string,
|
|
237
352
|
): Promise<StagedRecord | undefined> {
|
|
238
|
-
if (!
|
|
353
|
+
if (!isAttachmentName(file)) return undefined;
|
|
239
354
|
const stored = Bun.file(join(staging, RECORDS, `${file}.json`));
|
|
240
355
|
if (
|
|
241
356
|
!(await stored.exists()) ||
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import type { TurnAttachments } from "../domain/attachment.ts";
|
|
2
|
+
import { plainContentType, plainText } from "./attachment-name.ts";
|
|
3
|
+
|
|
4
|
+
/** A name a prompt can quote: no control or bidi characters, and no quote to close it early. */
|
|
5
|
+
const quotable = (name: string): string => plainText(name).replaceAll('"', "'");
|
|
2
6
|
|
|
3
7
|
function formatSize(bytes: number): string {
|
|
4
8
|
if (bytes < 1024) return `${bytes} B`;
|
|
@@ -19,13 +23,13 @@ export function withAttachmentsBlock(
|
|
|
19
23
|
for (const file of attachments.files) {
|
|
20
24
|
const origin = file.fromReference ? ", from the replied-to message" : "";
|
|
21
25
|
lines.push(
|
|
22
|
-
`- ${file.file} (original name "${file.name}", ${file.contentType}, ${formatSize(file.size)}${origin})`,
|
|
26
|
+
`- ${plainText(file.file)} (original name "${quotable(file.name)}", ${plainContentType(file.contentType)}, ${formatSize(file.size)}${origin})`,
|
|
23
27
|
);
|
|
24
28
|
}
|
|
25
29
|
for (const failure of attachments.failures) {
|
|
26
30
|
const origin = failure.fromReference ? " from the replied-to message" : "";
|
|
27
31
|
lines.push(
|
|
28
|
-
`- "${failure.name}"${origin} could not be received: ${failure.reason}`,
|
|
32
|
+
`- "${quotable(failure.name)}"${origin} could not be received: ${failure.reason}`,
|
|
29
33
|
);
|
|
30
34
|
}
|
|
31
35
|
if (lines.length === 0) return text;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { isAttachmentName, plainText } from "./attachment-name.ts";
|
|
2
3
|
import { RECORDS } from "./attachment-store.ts";
|
|
3
4
|
|
|
4
5
|
/** A file someone attached to the conversation, as a tool reads it. */
|
|
@@ -31,12 +32,13 @@ export async function openAttachment(
|
|
|
31
32
|
dir: string,
|
|
32
33
|
file: string,
|
|
33
34
|
): Promise<ToolAttachment> {
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
const shown = plainText(file);
|
|
36
|
+
if (!isAttachmentName(file))
|
|
37
|
+
throw new AttachmentLookupError(`"${shown}" is not an attachment name`);
|
|
36
38
|
const handle = Bun.file(join(dir, file));
|
|
37
39
|
if (!(await handle.exists()))
|
|
38
40
|
throw new AttachmentLookupError(
|
|
39
|
-
`no attachment named "${
|
|
41
|
+
`no attachment named "${shown}" in this channel`,
|
|
40
42
|
);
|
|
41
43
|
const record = Bun.file(join(dir, RECORDS, `${file}.json`));
|
|
42
44
|
const recorded: Recorded = (await record.exists()) ? await record.json() : {};
|
|
@@ -48,6 +50,13 @@ export async function openAttachment(
|
|
|
48
50
|
? recorded.contentType
|
|
49
51
|
: handle.type,
|
|
50
52
|
size: handle.size,
|
|
51
|
-
|
|
53
|
+
// A file system error names the path on the host; the model sees only the attachment's name.
|
|
54
|
+
bytes: async () => {
|
|
55
|
+
try {
|
|
56
|
+
return new Uint8Array(await handle.arrayBuffer());
|
|
57
|
+
} catch {
|
|
58
|
+
throw new AttachmentLookupError(`"${shown}" could not be read`);
|
|
59
|
+
}
|
|
60
|
+
},
|
|
52
61
|
};
|
|
53
62
|
}
|
|
@@ -19,7 +19,9 @@ export type AttachmentRefusalCode =
|
|
|
19
19
|
/** No saved file of that name is waiting for this person in this conversation. */
|
|
20
20
|
| "unknown_file"
|
|
21
21
|
/** The file is over the core's limit of 25 MiB. */
|
|
22
|
-
| "too_large"
|
|
22
|
+
| "too_large"
|
|
23
|
+
/** Using the files would take the principal past the `usedBytesLimit` the plugin passed. */
|
|
24
|
+
| "quota_exceeded";
|
|
23
25
|
|
|
24
26
|
/** An attachment call the store refused; `code` says why, and the plugin decides what its client sees. */
|
|
25
27
|
export class AttachmentRefusal extends Error {
|
|
@@ -32,6 +34,16 @@ export class AttachmentRefusal extends Error {
|
|
|
32
34
|
}
|
|
33
35
|
}
|
|
34
36
|
|
|
37
|
+
/** What a plugin asks of `turnAttachments` besides the files. */
|
|
38
|
+
export interface TurnAttachmentOptions {
|
|
39
|
+
/**
|
|
40
|
+
* The most bytes the principal may have used in turns across all their conversations, these
|
|
41
|
+
* files included. A call that would pass it is refused with `quota_exceeded` and moves nothing.
|
|
42
|
+
* Without it the core sets no limit; deleting a conversation gives its bytes back.
|
|
43
|
+
*/
|
|
44
|
+
usedBytesLimit?: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
35
47
|
/**
|
|
36
48
|
* Files a plugin takes from a person outside a turn, such as an upload over HTTP, kept for the
|
|
37
49
|
* conversation until a turn uses them. A saved file waits for its person: only the principal that
|
|
@@ -40,6 +52,13 @@ export class AttachmentRefusal extends Error {
|
|
|
40
52
|
* conversation's attachment directory, where the turn's `read_attachment` and
|
|
41
53
|
* `ToolTurn.attachment()` find it. Calls reject with `AttachmentRefusal` for a refused call, and
|
|
42
54
|
* throw a PluginError when the host has no `dataDir`.
|
|
55
|
+
*
|
|
56
|
+
* Who may use a channel is the plugin's to check, not the port's. The port refuses another
|
|
57
|
+
* principal only in a conversation the host's registry records as private to someone else; for a
|
|
58
|
+
* channel the registry does not know, or when the host has no registry, it admits every principal.
|
|
59
|
+
* A plugin whose channel key comes from a person must first check that the channel is that
|
|
60
|
+
* person's, as the web chat does before every call; the plugin that owns a channel checks it,
|
|
61
|
+
* because only that plugin knows what the channel is.
|
|
43
62
|
*/
|
|
44
63
|
export interface AttachmentPort {
|
|
45
64
|
/** Keeps a file for the principal in the conversation until a turn uses it. */
|
|
@@ -51,12 +70,14 @@ export interface AttachmentPort {
|
|
|
51
70
|
/**
|
|
52
71
|
* The turn's attachments for files `save` returned: each moves into the conversation, up to four
|
|
53
72
|
* images are prepared for the model, and every file is named in the turn's `## Attachments`
|
|
54
|
-
* block. Refuses the whole set when one file is unknown or already used
|
|
73
|
+
* block. Refuses the whole set when one file is unknown or already used, or when
|
|
74
|
+
* `options.usedBytesLimit` would be passed; none moves then, however many calls run at once.
|
|
55
75
|
*/
|
|
56
76
|
turnAttachments(
|
|
57
77
|
channel: ChannelKey,
|
|
58
78
|
principalId: string,
|
|
59
79
|
files: readonly string[],
|
|
80
|
+
options?: TurnAttachmentOptions,
|
|
60
81
|
): Promise<TurnAttachments>;
|
|
61
82
|
/** Discards a saved file no turn used; false when there is none. */
|
|
62
83
|
remove(
|
|
@@ -3,7 +3,10 @@ import {
|
|
|
3
3
|
type ModelRuntime,
|
|
4
4
|
SessionManager,
|
|
5
5
|
} from "@earendil-works/pi-coding-agent";
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
discardConversationFiles,
|
|
8
|
+
ownerAttachmentDir,
|
|
9
|
+
} from "../attachments/attachment-dir.ts";
|
|
7
10
|
import { withAttachmentsBlock } from "../attachments/prompt-block.ts";
|
|
8
11
|
import type { AgentRuntime } from "../contract/runtime.ts";
|
|
9
12
|
import { NO_ATTACHMENTS, type TurnAttachments } from "../domain/attachment.ts";
|
|
@@ -447,6 +450,7 @@ export class PiAgentRuntime implements AgentRuntime {
|
|
|
447
450
|
async deleteConversation(channel: ChannelKey): Promise<void> {
|
|
448
451
|
await this.#sessions.forget(channel);
|
|
449
452
|
rmSync(this.#factory.sessionDir(channel), { recursive: true, force: true });
|
|
453
|
+
await discardConversationFiles(this.#options.dataDir, channel);
|
|
450
454
|
this.#options.logger.info({ channel }, "conversation deleted");
|
|
451
455
|
}
|
|
452
456
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { join } from "node:path";
|
|
2
2
|
import { extractText, getDocumentProxy } from "unpdf";
|
|
3
|
+
import { isAttachmentName, plainText } from "../attachments/attachment-name.ts";
|
|
3
4
|
|
|
4
5
|
/** Characters returned per read; the model can ask for a later offset. */
|
|
5
6
|
export const READ_LIMIT = 60_000;
|
|
@@ -66,14 +67,20 @@ export async function readAttachment(
|
|
|
66
67
|
file: string,
|
|
67
68
|
offset = 0,
|
|
68
69
|
): Promise<ReadResult> {
|
|
69
|
-
if (
|
|
70
|
-
throw new Error(`"${file}" is not an attachment name`);
|
|
70
|
+
if (!isAttachmentName(file)) {
|
|
71
|
+
throw new Error(`"${plainText(file)}" is not an attachment name`);
|
|
71
72
|
}
|
|
72
73
|
const handle = Bun.file(join(dir, file));
|
|
73
74
|
if (!(await handle.exists())) {
|
|
74
75
|
throw new Error(`no attachment named "${file}" in this channel`);
|
|
75
76
|
}
|
|
76
|
-
|
|
77
|
+
let bytes: Uint8Array;
|
|
78
|
+
try {
|
|
79
|
+
bytes = new Uint8Array(await handle.arrayBuffer());
|
|
80
|
+
} catch {
|
|
81
|
+
// A file system error names the path on the host; the model sees only the attachment's name.
|
|
82
|
+
throw new Error(`"${plainText(file)}" could not be read`);
|
|
83
|
+
}
|
|
77
84
|
let text: string;
|
|
78
85
|
if (isPdf(file, bytes)) {
|
|
79
86
|
const pdf = await getDocumentProxy(bytes);
|
package/src/index.ts
CHANGED