pi-roundtable 0.17.3 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/docs/plugins.md +93 -5
- package/package.json +8 -6
- package/src/core/builtin/discord.ts +11 -0
- package/src/core/discord/discord-surface.ts +8 -0
- package/src/core/discord/poster/poster-checks.ts +272 -0
- package/src/core/discord/poster/poster-errors.ts +106 -0
- package/src/core/discord/poster/poster.ts +301 -0
- package/src/core/discord/poster/rest-poster-core.ts +112 -0
- package/src/core/discord/poster/rest-poster-guild.ts +329 -0
- package/src/core/discord/poster/rest-poster.ts +488 -0
- package/src/core/images/codex-image.ts +257 -0
- package/src/core/images/codex-images-plugin.ts +76 -0
- package/src/core/modules/delegation/web-research-worker.ts +1 -1
- package/src/core/ops/alert-throttle.ts +81 -0
- package/src/core/ops/error-reporter.ts +20 -17
- package/src/core/ops/ops-alerts.ts +132 -0
- package/src/core/testing/fake-poster-guild.ts +491 -0
- package/src/core/testing/fake-poster.ts +475 -0
- package/src/core/testing/test-host.ts +16 -5
- package/src/discord/index.ts +21 -0
- package/src/kit/images.ts +15 -0
- package/src/kit/index.ts +21 -0
- package/src/kit/jev-judge.ts +153 -0
- package/src/kit/ops-alerts.ts +9 -0
- package/src/testing.ts +15 -1
- package/templates/adapters/web/package.json.tmpl +2 -2
- package/templates/official/codex-images/plugin.test.ts.tmpl +31 -0
- package/templates/official/codex-images/plugin.ts +8 -232
- package/templates/package.json.tmpl +2 -2
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
// A client for ChatGPT's undocumented Codex backend, signed in with an openai-codex login, that draws one image.
|
|
2
|
+
|
|
3
|
+
import type { ReferenceImage } from "../contract/providers.ts";
|
|
4
|
+
|
|
5
|
+
const CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
6
|
+
const JWT_CLAIM_PATH = "https://api.openai.com/auth";
|
|
7
|
+
const REQUEST_TIMEOUT_MS = 5 * 60_000;
|
|
8
|
+
const MAX_IMAGE_BYTES = 32 * 1024 * 1024;
|
|
9
|
+
const MAX_IMAGE_BASE64 = Math.ceil(MAX_IMAGE_BYTES / 3) * 4;
|
|
10
|
+
|
|
11
|
+
/** The Pi provider whose login pays for the images. */
|
|
12
|
+
export const CODEX_IMAGES_PROVIDER = "openai-codex";
|
|
13
|
+
/** Routes the Codex request; the backend picks the image model itself. */
|
|
14
|
+
export const CODEX_IMAGE_MODEL = "gpt-6-sol";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Codex ran the image tool but returned no image, or answered without one; usually it refused
|
|
18
|
+
* the prompt. Asking again with the same prompt fails again.
|
|
19
|
+
*/
|
|
20
|
+
export class ImageNotGeneratedError extends Error {
|
|
21
|
+
override name = "ImageNotGeneratedError";
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** The part of `fetch` the client uses, so a test can stand in for the network. `typeof fetch` is assignable. */
|
|
25
|
+
export type CodexFetch = (url: string, init: RequestInit) => Promise<Response>;
|
|
26
|
+
|
|
27
|
+
export interface CodexImageRequest {
|
|
28
|
+
/** The openai-codex login's access token; read it again for each drawing, because the login refreshes. */
|
|
29
|
+
token: string;
|
|
30
|
+
prompt: string;
|
|
31
|
+
references?: readonly ReferenceImage[];
|
|
32
|
+
/** The model that routes the request; default `CODEX_IMAGE_MODEL`. */
|
|
33
|
+
model?: string;
|
|
34
|
+
/** Aborts the request and the read of its stream; the five-minute limit applies as well. */
|
|
35
|
+
signal?: AbortSignal;
|
|
36
|
+
/** The request function; default the global `fetch`. */
|
|
37
|
+
fetch?: CodexFetch;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface CodexImage {
|
|
41
|
+
/** PNG bytes. */
|
|
42
|
+
data: Uint8Array;
|
|
43
|
+
/** The prompt Codex says it drew, when it tells. */
|
|
44
|
+
revisedPrompt?: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** What bounds one drawing; the core's tests lower these, the public client always uses the defaults. */
|
|
48
|
+
export interface CodexImageLimits {
|
|
49
|
+
timeoutMs: number;
|
|
50
|
+
maxStreamBytes: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export const CODEX_IMAGE_LIMITS: CodexImageLimits = {
|
|
54
|
+
timeoutMs: REQUEST_TIMEOUT_MS,
|
|
55
|
+
maxStreamBytes: MAX_IMAGE_BYTES,
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/** The ChatGPT account the login token belongs to, from the token's claims. */
|
|
59
|
+
function chatGptAccountId(token: string): string {
|
|
60
|
+
const [, payload] = token.split(".");
|
|
61
|
+
if (!payload) throw new Error("the Codex token is not a JWT");
|
|
62
|
+
let claims: Record<string, unknown>;
|
|
63
|
+
try {
|
|
64
|
+
claims = JSON.parse(Buffer.from(payload, "base64url").toString("utf8"));
|
|
65
|
+
} catch {
|
|
66
|
+
throw new Error("the Codex token's claims are not JSON");
|
|
67
|
+
}
|
|
68
|
+
const auth = claims[JWT_CLAIM_PATH] as
|
|
69
|
+
| { chatgpt_account_id?: unknown }
|
|
70
|
+
| undefined;
|
|
71
|
+
const id = auth?.chatgpt_account_id;
|
|
72
|
+
if (typeof id !== "string" || !id)
|
|
73
|
+
throw new Error("the Codex token has no ChatGPT account");
|
|
74
|
+
return id;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function object(value: unknown): Record<string, unknown> {
|
|
78
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
79
|
+
? (value as Record<string, unknown>)
|
|
80
|
+
: {};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The one image an `image_generation_call` item carries; anything else is not an image. */
|
|
84
|
+
function imageOf(value: unknown): CodexImage | undefined {
|
|
85
|
+
const item = object(value);
|
|
86
|
+
if (item.type !== "image_generation_call") return undefined;
|
|
87
|
+
if (
|
|
88
|
+
item.status !== "completed" ||
|
|
89
|
+
typeof item.result !== "string" ||
|
|
90
|
+
!item.result
|
|
91
|
+
) {
|
|
92
|
+
const { result: _result, ...detail } = item;
|
|
93
|
+
throw new ImageNotGeneratedError(
|
|
94
|
+
`Codex image generation did not complete (status ${String(item.status)}): ${JSON.stringify(detail).slice(0, 300)}`,
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
if (item.result.length > MAX_IMAGE_BASE64)
|
|
98
|
+
throw new Error("the image exceeds 32 MiB");
|
|
99
|
+
const data = new Uint8Array(Buffer.from(item.result, "base64"));
|
|
100
|
+
return typeof item.revised_prompt === "string" && item.revised_prompt
|
|
101
|
+
? { data, revisedPrompt: item.revised_prompt }
|
|
102
|
+
: { data };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The frames of a server-sent event stream: the text between blank lines. */
|
|
106
|
+
async function* frames(
|
|
107
|
+
body: ReadableStream<Uint8Array>,
|
|
108
|
+
signal: AbortSignal,
|
|
109
|
+
maxBytes: number,
|
|
110
|
+
): AsyncGenerator<string> {
|
|
111
|
+
const decoder = new TextDecoder();
|
|
112
|
+
let buffer = "";
|
|
113
|
+
let read = 0;
|
|
114
|
+
for await (const chunk of body) {
|
|
115
|
+
signal.throwIfAborted();
|
|
116
|
+
read += chunk.byteLength;
|
|
117
|
+
if (read > maxBytes) throw new Error("Codex image response too large");
|
|
118
|
+
buffer += decoder.decode(chunk, { stream: true });
|
|
119
|
+
let match = /\r?\n\r?\n/.exec(buffer);
|
|
120
|
+
while (match) {
|
|
121
|
+
yield buffer.slice(0, match.index);
|
|
122
|
+
buffer = buffer.slice(match.index + match[0].length);
|
|
123
|
+
match = /\r?\n\r?\n/.exec(buffer);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (buffer.trim()) yield buffer;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The event a frame carries, or undefined for a frame without data. */
|
|
130
|
+
function eventOf(frame: string): Record<string, unknown> | undefined {
|
|
131
|
+
const data = frame
|
|
132
|
+
.split(/\r?\n/)
|
|
133
|
+
.flatMap((line) => (line.startsWith("data:") ? [line.slice(5).trim()] : []))
|
|
134
|
+
.join("\n");
|
|
135
|
+
if (!data || data === "[DONE]") return undefined;
|
|
136
|
+
try {
|
|
137
|
+
return object(JSON.parse(data));
|
|
138
|
+
} catch {
|
|
139
|
+
throw new Error(`Codex sent a malformed event: ${data.slice(0, 200)}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Reads the events until the response completes, and returns its first image. */
|
|
144
|
+
async function parseImageStream(
|
|
145
|
+
response: Response,
|
|
146
|
+
signal: AbortSignal,
|
|
147
|
+
maxBytes: number,
|
|
148
|
+
): Promise<CodexImage> {
|
|
149
|
+
if (!response.body) throw new Error("Codex returned no body");
|
|
150
|
+
let image: CodexImage | undefined;
|
|
151
|
+
for await (const frame of frames(response.body, signal, maxBytes)) {
|
|
152
|
+
const event = eventOf(frame);
|
|
153
|
+
switch (event?.type) {
|
|
154
|
+
case "error":
|
|
155
|
+
case "response.failed": {
|
|
156
|
+
const error = object(object(event.response).error ?? event.error);
|
|
157
|
+
throw new Error(
|
|
158
|
+
`Codex failed: ${String(error.message ?? error.code ?? "unknown error")}`,
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
case "response.incomplete":
|
|
162
|
+
throw new Error("the Codex response was incomplete");
|
|
163
|
+
case "response.output_item.done":
|
|
164
|
+
image ??= imageOf(event.item);
|
|
165
|
+
break;
|
|
166
|
+
case "response.completed": {
|
|
167
|
+
const output = object(event.response).output;
|
|
168
|
+
if (Array.isArray(output))
|
|
169
|
+
for (const item of output) image ??= imageOf(item);
|
|
170
|
+
if (!image)
|
|
171
|
+
throw new ImageNotGeneratedError("Codex answered without an image");
|
|
172
|
+
return image;
|
|
173
|
+
}
|
|
174
|
+
default:
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
throw new Error("the Codex stream ended before completion");
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function send(
|
|
182
|
+
request: CodexImageRequest,
|
|
183
|
+
signal: AbortSignal,
|
|
184
|
+
maxBytes: number,
|
|
185
|
+
): Promise<CodexImage> {
|
|
186
|
+
const fetchImpl = request.fetch ?? fetch;
|
|
187
|
+
const response = await fetchImpl(CODEX_RESPONSES_URL, {
|
|
188
|
+
method: "POST",
|
|
189
|
+
redirect: "error",
|
|
190
|
+
signal,
|
|
191
|
+
headers: {
|
|
192
|
+
authorization: `Bearer ${request.token}`,
|
|
193
|
+
"chatgpt-account-id": chatGptAccountId(request.token),
|
|
194
|
+
originator: "pi",
|
|
195
|
+
"openai-beta": "responses=experimental",
|
|
196
|
+
accept: "text/event-stream",
|
|
197
|
+
"content-type": "application/json",
|
|
198
|
+
},
|
|
199
|
+
body: JSON.stringify({
|
|
200
|
+
model: request.model ?? CODEX_IMAGE_MODEL,
|
|
201
|
+
store: false,
|
|
202
|
+
stream: true,
|
|
203
|
+
instructions:
|
|
204
|
+
"You are generating bitmap image assets. For this request, call the image_generation tool exactly once. Do not answer with only text unless image generation is unavailable.",
|
|
205
|
+
input: [
|
|
206
|
+
{
|
|
207
|
+
role: "user",
|
|
208
|
+
content: [
|
|
209
|
+
{ type: "input_text", text: request.prompt },
|
|
210
|
+
...(request.references ?? []).map((image) => ({
|
|
211
|
+
type: "input_image",
|
|
212
|
+
image_url: `data:${image.mimeType};base64,${image.data}`,
|
|
213
|
+
})),
|
|
214
|
+
],
|
|
215
|
+
},
|
|
216
|
+
],
|
|
217
|
+
tools: [{ type: "image_generation", output_format: "png" }],
|
|
218
|
+
tool_choice: "auto",
|
|
219
|
+
parallel_tool_calls: false,
|
|
220
|
+
text: { verbosity: "low" },
|
|
221
|
+
}),
|
|
222
|
+
});
|
|
223
|
+
if (!response.ok) {
|
|
224
|
+
const detail = (await response.text()).slice(0, 300);
|
|
225
|
+
throw new Error(`Codex answered ${response.status}: ${detail}`);
|
|
226
|
+
}
|
|
227
|
+
return parseImageStream(response, signal, maxBytes);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** `drawCodexImage` with its limits given, for the core's tests. */
|
|
231
|
+
export async function requestCodexImage(
|
|
232
|
+
request: CodexImageRequest,
|
|
233
|
+
limits: CodexImageLimits,
|
|
234
|
+
): Promise<CodexImage> {
|
|
235
|
+
const timeout = AbortSignal.timeout(limits.timeoutMs);
|
|
236
|
+
const signal = request.signal
|
|
237
|
+
? AbortSignal.any([request.signal, timeout])
|
|
238
|
+
: timeout;
|
|
239
|
+
signal.throwIfAborted();
|
|
240
|
+
try {
|
|
241
|
+
return await send(request, signal, limits.maxStreamBytes);
|
|
242
|
+
} catch (error) {
|
|
243
|
+
// The abort reaches the caller as its own reason, not as whatever the stream wrapped it in.
|
|
244
|
+
if (signal.aborted) throw signal.reason;
|
|
245
|
+
throw error;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Draws one image through the Codex backend. It aborts with `request.signal`, before the response
|
|
251
|
+
* and while its stream is read, stops after five minutes, and refuses a stream over 32 MiB.
|
|
252
|
+
*/
|
|
253
|
+
export function drawCodexImage(
|
|
254
|
+
request: CodexImageRequest,
|
|
255
|
+
): Promise<CodexImage> {
|
|
256
|
+
return requestCodexImage(request, CODEX_IMAGE_LIMITS);
|
|
257
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// The codex-images plugin: fills the images slot with drawCodexImage.
|
|
2
|
+
|
|
3
|
+
import { definePlugin } from "../define.ts";
|
|
4
|
+
import { PluginError } from "../errors.ts";
|
|
5
|
+
import type { RoundtablePlugin } from "../plugin.ts";
|
|
6
|
+
import {
|
|
7
|
+
CODEX_IMAGE_MODEL,
|
|
8
|
+
CODEX_IMAGES_PROVIDER,
|
|
9
|
+
type CodexFetch,
|
|
10
|
+
drawCodexImage,
|
|
11
|
+
} from "./codex-image.ts";
|
|
12
|
+
|
|
13
|
+
export interface CodexImagesOptions {
|
|
14
|
+
/** The request function; default the global `fetch`. */
|
|
15
|
+
fetch?: CodexFetch;
|
|
16
|
+
/** The model that routes the request; default `CODEX_IMAGE_MODEL`. */
|
|
17
|
+
model?: string;
|
|
18
|
+
/**
|
|
19
|
+
* What setup does when the openai-codex login is missing or unusable: "throw" (the default)
|
|
20
|
+
* stops the host; "warn" logs a warning, and a drawing then fails with the same PluginError.
|
|
21
|
+
*/
|
|
22
|
+
onMissingLogin?: "throw" | "warn";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const NO_LOGIN = `the host has no login for ${CODEX_IMAGES_PROVIDER}, the provider that pays for the images. Log in to the ${CODEX_IMAGES_PROVIDER} provider with Pi so the agent directory's auth.json holds it, or remove this plugin.`;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The `codex-images` plugin. It fills `providers.images`, reads the login's token for each drawing
|
|
29
|
+
* through `context.apiKey`, and aborts a drawing with the turn that asked for it.
|
|
30
|
+
*/
|
|
31
|
+
export function createCodexImages(
|
|
32
|
+
options: CodexImagesOptions = {},
|
|
33
|
+
): RoundtablePlugin {
|
|
34
|
+
const {
|
|
35
|
+
fetch: fetchImpl,
|
|
36
|
+
model = CODEX_IMAGE_MODEL,
|
|
37
|
+
onMissingLogin = "throw",
|
|
38
|
+
} = options;
|
|
39
|
+
let apiKey: ((provider: string) => Promise<string | undefined>) | undefined;
|
|
40
|
+
// The login refreshes its token, so each image reads it again instead of keeping the first.
|
|
41
|
+
const token = async (): Promise<string> => {
|
|
42
|
+
const value = await apiKey?.(CODEX_IMAGES_PROVIDER);
|
|
43
|
+
if (!value) throw new PluginError(`plugin codex-images: ${NO_LOGIN}`);
|
|
44
|
+
return value;
|
|
45
|
+
};
|
|
46
|
+
return definePlugin({
|
|
47
|
+
name: "codex-images",
|
|
48
|
+
providers: {
|
|
49
|
+
// The images slot draws an agent's avatar from a prompt and reference pictures.
|
|
50
|
+
images: async (prompt, references, drawing) =>
|
|
51
|
+
(
|
|
52
|
+
await drawCodexImage({
|
|
53
|
+
token: await token(),
|
|
54
|
+
prompt,
|
|
55
|
+
references,
|
|
56
|
+
model,
|
|
57
|
+
signal: drawing?.signal,
|
|
58
|
+
fetch: fetchImpl,
|
|
59
|
+
})
|
|
60
|
+
).data,
|
|
61
|
+
},
|
|
62
|
+
async setup(context) {
|
|
63
|
+
apiKey = context.apiKey;
|
|
64
|
+
try {
|
|
65
|
+
await token();
|
|
66
|
+
} catch (error) {
|
|
67
|
+
if (onMissingLogin !== "warn" || !(error instanceof PluginError))
|
|
68
|
+
throw error;
|
|
69
|
+
context.logger.warn(
|
|
70
|
+
`${NO_LOGIN} Drawing images fails until the login exists.`,
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
return {};
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
}
|
|
@@ -27,7 +27,7 @@ function webAccessDir(): string {
|
|
|
27
27
|
return packageDir("pi-web-access");
|
|
28
28
|
} catch (error) {
|
|
29
29
|
throw new PluginError(
|
|
30
|
-
"the delegation worker loads pi-web-access, which this project does not have installed. It is a peer dependency of pi-roundtable: run `bun add pi-web-access@0.
|
|
30
|
+
"the delegation worker loads pi-web-access, which this project does not have installed. It is a peer dependency of pi-roundtable: run `bun add pi-web-access@0.40.0` (or your own build of it, which the worker then uses too).",
|
|
31
31
|
{ cause: error },
|
|
32
32
|
);
|
|
33
33
|
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/** What `admit` finds: the key is held, its group is full, or the alert may go. */
|
|
2
|
+
export type Admission = "open" | "repeat" | "full";
|
|
3
|
+
|
|
4
|
+
export interface AlertThrottleOptions {
|
|
5
|
+
/** How long a key, and a slot of its group, is held. */
|
|
6
|
+
windowMs: number;
|
|
7
|
+
/** The most slots one group holds inside the window. */
|
|
8
|
+
groupCap: number;
|
|
9
|
+
now: () => number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The rolling window behind the error reports and the ops alerts: a key is held for a window, and
|
|
14
|
+
* a group of keys has a fixed number of slots inside it. Nothing here posts or schedules; the
|
|
15
|
+
* caller asks, then holds and takes a slot in the same synchronous step, so two alerts raised
|
|
16
|
+
* together for one key go out once.
|
|
17
|
+
*/
|
|
18
|
+
export class AlertThrottle {
|
|
19
|
+
readonly #windowMs: number;
|
|
20
|
+
readonly #groupCap: number;
|
|
21
|
+
readonly #now: () => number;
|
|
22
|
+
readonly #held = new Map<string, number>();
|
|
23
|
+
/** The times each group took a slot inside the window, oldest first. */
|
|
24
|
+
readonly #slots = new Map<string, number[]>();
|
|
25
|
+
|
|
26
|
+
constructor(options: AlertThrottleOptions) {
|
|
27
|
+
this.#windowMs = options.windowMs;
|
|
28
|
+
this.#groupCap = options.groupCap;
|
|
29
|
+
this.#now = options.now;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** "repeat" when the key is held, else "full" when the group is (unless `uncapped`), else "open". */
|
|
33
|
+
admit(key: string, group: string, uncapped = false): Admission {
|
|
34
|
+
const now = this.#now();
|
|
35
|
+
this.#pruneKeys(now);
|
|
36
|
+
if (this.#held.has(key)) return "repeat";
|
|
37
|
+
if (!uncapped && this.#live(group, now).length >= this.#groupCap)
|
|
38
|
+
return "full";
|
|
39
|
+
return "open";
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Holds the key for a window. */
|
|
43
|
+
hold(key: string): void {
|
|
44
|
+
this.#held.set(key, this.#now());
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Takes one slot of the group for a window. */
|
|
48
|
+
slot(group: string): void {
|
|
49
|
+
const now = this.#now();
|
|
50
|
+
this.#live(group, now).push(now);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Gives back the key and the group's newest slot (a post that failed). */
|
|
54
|
+
release(key: string, group?: string): void {
|
|
55
|
+
this.#held.delete(key);
|
|
56
|
+
if (group !== undefined) this.#slots.get(group)?.pop();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** When the group's oldest slot leaves the window; undefined while it has room. */
|
|
60
|
+
roomAt(group: string): number | undefined {
|
|
61
|
+
const slots = this.#live(group, this.#now());
|
|
62
|
+
const oldest = slots[0];
|
|
63
|
+
return slots.length >= this.#groupCap && oldest !== undefined
|
|
64
|
+
? oldest + this.#windowMs
|
|
65
|
+
: undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
#pruneKeys(now: number): void {
|
|
69
|
+
for (const [key, at] of this.#held)
|
|
70
|
+
if (now - at >= this.#windowMs) this.#held.delete(key);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** The group's slots still inside the window; the array is kept. */
|
|
74
|
+
#live(group: string, now: number): number[] {
|
|
75
|
+
const live = (this.#slots.get(group) ?? []).filter(
|
|
76
|
+
(at) => now - at < this.#windowMs,
|
|
77
|
+
);
|
|
78
|
+
this.#slots.set(group, live);
|
|
79
|
+
return live;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -2,6 +2,7 @@ import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
2
2
|
import type { ScheduledOutcome } from "../contract/channels.ts";
|
|
3
3
|
import type { LogEntry, Logger } from "../log.ts";
|
|
4
4
|
import type { ChannelKey } from "../sessions.ts";
|
|
5
|
+
import { AlertThrottle } from "./alert-throttle.ts";
|
|
5
6
|
|
|
6
7
|
/** Errors that are expected and never reported, matched against the message and the error's. */
|
|
7
8
|
export const IGNORED_ERRORS: readonly RegExp[] = [];
|
|
@@ -9,6 +10,8 @@ export const IGNORED_ERRORS: readonly RegExp[] = [];
|
|
|
9
10
|
const HOUR_MS = 3_600_000;
|
|
10
11
|
/** Reports, summaries included, per rolling hour. */
|
|
11
12
|
const REPORTS_PER_HOUR = 5;
|
|
13
|
+
/** The throttle group of every report. */
|
|
14
|
+
const ALL = "*";
|
|
12
15
|
/** Errors kept from before the agent server is ready; older ones are dropped and counted. */
|
|
13
16
|
const EARLY_BUFFER = 20;
|
|
14
17
|
const REPORT_LIMIT = 1500;
|
|
@@ -70,8 +73,8 @@ export class ErrorReporter {
|
|
|
70
73
|
#earlyDropped = 0;
|
|
71
74
|
/** Report turns started and not yet finished. */
|
|
72
75
|
#turns = 0;
|
|
73
|
-
|
|
74
|
-
readonly #
|
|
76
|
+
/** One group for every report: the limit is on the process, not on a kind of error. */
|
|
77
|
+
readonly #throttle: AlertThrottle;
|
|
75
78
|
readonly #suppressed = new Map<string, number>();
|
|
76
79
|
#summaryDue = false;
|
|
77
80
|
|
|
@@ -80,6 +83,11 @@ export class ErrorReporter {
|
|
|
80
83
|
this.#app = options.app;
|
|
81
84
|
this.#ignored = options.ignored ?? IGNORED_ERRORS;
|
|
82
85
|
this.#now = options.now ?? Date.now;
|
|
86
|
+
this.#throttle = new AlertThrottle({
|
|
87
|
+
windowMs: HOUR_MS,
|
|
88
|
+
groupCap: REPORTS_PER_HOUR,
|
|
89
|
+
now: this.#now,
|
|
90
|
+
});
|
|
83
91
|
this.#setTimer =
|
|
84
92
|
options.setTimer ??
|
|
85
93
|
((run, ms) => {
|
|
@@ -134,43 +142,38 @@ export class ErrorReporter {
|
|
|
134
142
|
#consider(entry: LogEntry): void {
|
|
135
143
|
const now = this.#now();
|
|
136
144
|
const key = fingerprint(entry);
|
|
137
|
-
const
|
|
138
|
-
if (
|
|
139
|
-
if (
|
|
145
|
+
const admission = this.#throttle.admit(key, ALL);
|
|
146
|
+
if (admission === "repeat") return;
|
|
147
|
+
if (admission === "full") {
|
|
140
148
|
this.#suppressed.set(key, (this.#suppressed.get(key) ?? 0) + 1);
|
|
141
149
|
this.#scheduleSummary(now);
|
|
142
150
|
return;
|
|
143
151
|
}
|
|
144
|
-
this.#
|
|
145
|
-
this.#
|
|
152
|
+
this.#throttle.hold(key);
|
|
153
|
+
this.#throttle.slot(ALL);
|
|
146
154
|
void this.#deliver(reportText(entry, this.#app));
|
|
147
155
|
}
|
|
148
156
|
|
|
149
|
-
#hasRoom(now: number): boolean {
|
|
150
|
-
this.#sent = this.#sent.filter((at) => now - at < HOUR_MS);
|
|
151
|
-
return this.#sent.length < REPORTS_PER_HOUR;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
157
|
#scheduleSummary(now: number): void {
|
|
155
158
|
if (this.#summaryDue) return;
|
|
156
159
|
this.#summaryDue = true;
|
|
157
|
-
const
|
|
158
|
-
this.#setTimer(() => this.#summarize(),
|
|
160
|
+
const roomAt = this.#throttle.roomAt(ALL) ?? now;
|
|
161
|
+
this.#setTimer(() => this.#summarize(), roomAt - now);
|
|
159
162
|
}
|
|
160
163
|
|
|
161
164
|
#summarize(): void {
|
|
162
165
|
this.#summaryDue = false;
|
|
163
166
|
const now = this.#now();
|
|
164
167
|
if (this.#suppressed.size === 0) return;
|
|
165
|
-
if (
|
|
168
|
+
if (this.#throttle.roomAt(ALL) !== undefined) {
|
|
166
169
|
this.#scheduleSummary(now);
|
|
167
170
|
return;
|
|
168
171
|
}
|
|
169
172
|
const counts = [...this.#suppressed];
|
|
170
173
|
this.#suppressed.clear();
|
|
171
174
|
const total = counts.reduce((sum, [, n]) => sum + n, 0);
|
|
172
|
-
for (const [key] of counts) this.#
|
|
173
|
-
this.#
|
|
175
|
+
for (const [key] of counts) this.#throttle.hold(key);
|
|
176
|
+
this.#throttle.slot(ALL);
|
|
174
177
|
const list = counts
|
|
175
178
|
.map(([key, n]) => (n > 1 ? `${key} (×${n})` : key))
|
|
176
179
|
.join("; ");
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import type { Logger } from "../log.ts";
|
|
2
|
+
import { AlertThrottle } from "./alert-throttle.ts";
|
|
3
|
+
|
|
4
|
+
const DEFAULT_WINDOW_MS = 60 * 60_000;
|
|
5
|
+
const DEFAULT_KIND_CAP = 8;
|
|
6
|
+
|
|
7
|
+
export interface OpsAlertsOptions {
|
|
8
|
+
/**
|
|
9
|
+
* Posts one alert's text where the people who run the host read it. Resolves true when it was
|
|
10
|
+
* posted and false when the sink chose not to (it logs why itself); a throw is a failed post.
|
|
11
|
+
* Either way the key and the slot are given back, so the next raise tries again. Absent: alerts are
|
|
12
|
+
* only recorded and logged, the key still held so the log has one line per window.
|
|
13
|
+
*/
|
|
14
|
+
sink?: (text: string) => Promise<boolean>;
|
|
15
|
+
logger: Logger;
|
|
16
|
+
/** What `faults()` calls each kind; a kind not listed is called by itself. */
|
|
17
|
+
kinds?: Readonly<Record<string, string>>;
|
|
18
|
+
/** Default one hour. */
|
|
19
|
+
windowMs?: number;
|
|
20
|
+
/** The most alerts of one kind posted inside a window, whatever their keys; default 8. */
|
|
21
|
+
kindCap?: number;
|
|
22
|
+
now?: () => number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface OpsRaiseOptions {
|
|
26
|
+
/** A short code (letters, digits, `_`, `:`) shown in `faults()`; never a name or anyone's words. */
|
|
27
|
+
code?: string;
|
|
28
|
+
/** News, not a fault: posted at most once per key, not counted against the kind's cap, not listed. */
|
|
29
|
+
info?: boolean;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The last fault of one kind since the start, and how often it came. */
|
|
33
|
+
export interface OpsAlertKind {
|
|
34
|
+
kind: string;
|
|
35
|
+
name: string;
|
|
36
|
+
at: number;
|
|
37
|
+
code: string | undefined;
|
|
38
|
+
/** Raised since the start, posted or not. */
|
|
39
|
+
raised: number;
|
|
40
|
+
/** Of those, held back because the key or the kind had been raised inside the window. */
|
|
41
|
+
suppressed: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** The kind of an alert is the word before the first colon of its key. */
|
|
45
|
+
function kindOf(key: string): string {
|
|
46
|
+
const colon = key.indexOf(":");
|
|
47
|
+
return colon === -1 ? key : key.slice(0, colon);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Short alerts for the people who run a host. Every alert has a key; its kind is the key's word
|
|
52
|
+
* before the first colon. A key is posted at most once a window and a kind at most `kindCap` times,
|
|
53
|
+
* so a fault that touches everything cannot flood the sink; what is held back is counted, never
|
|
54
|
+
* summed up in a post. An alert is posted when it is raised, not queued. A failed post is logged by
|
|
55
|
+
* the error's name and never thrown.
|
|
56
|
+
*/
|
|
57
|
+
export class OpsAlerts {
|
|
58
|
+
readonly #options: OpsAlertsOptions;
|
|
59
|
+
readonly #throttle: AlertThrottle;
|
|
60
|
+
readonly #kinds = new Map<string, OpsAlertKind>();
|
|
61
|
+
|
|
62
|
+
constructor(options: OpsAlertsOptions) {
|
|
63
|
+
this.#options = options;
|
|
64
|
+
this.#throttle = new AlertThrottle({
|
|
65
|
+
windowMs: options.windowMs ?? DEFAULT_WINDOW_MS,
|
|
66
|
+
groupCap: options.kindCap ?? DEFAULT_KIND_CAP,
|
|
67
|
+
now: options.now ?? Date.now,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Whether alerts reach a sink. */
|
|
72
|
+
get posts(): boolean {
|
|
73
|
+
return this.#options.sink !== undefined;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The last fault of each kind, newest first; memory only. */
|
|
77
|
+
faults(): OpsAlertKind[] {
|
|
78
|
+
return [...this.#kinds.values()].sort((a, b) => b.at - a.at);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Posts the alert unless its key or its kind is held inside the window; whether it was posted. */
|
|
82
|
+
async raise(
|
|
83
|
+
key: string,
|
|
84
|
+
text: string,
|
|
85
|
+
options: OpsRaiseOptions = {},
|
|
86
|
+
): Promise<boolean> {
|
|
87
|
+
const { sink, logger } = this.#options;
|
|
88
|
+
const kind = kindOf(key);
|
|
89
|
+
const info = options.info === true;
|
|
90
|
+
const admission = this.#throttle.admit(key, kind, info);
|
|
91
|
+
if (!info) this.#record(kind, options.code, admission !== "open");
|
|
92
|
+
if (admission !== "open") return false;
|
|
93
|
+
// Taken before the post, so two alerts raised together for one key post once.
|
|
94
|
+
this.#throttle.hold(key);
|
|
95
|
+
if (sink === undefined) {
|
|
96
|
+
// Nowhere to post: the key stays held so that the log has one line per window, not one per event.
|
|
97
|
+
logger.warn(
|
|
98
|
+
{ kind, ...(options.code ? { code: options.code } : {}) },
|
|
99
|
+
"an ops alert was raised and there is no sink to post it to",
|
|
100
|
+
);
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
if (!info) this.#throttle.slot(kind);
|
|
104
|
+
const undo = () => this.#throttle.release(key, info ? undefined : kind);
|
|
105
|
+
try {
|
|
106
|
+
if (await sink(text)) return true;
|
|
107
|
+
undo();
|
|
108
|
+
return false;
|
|
109
|
+
} catch (error) {
|
|
110
|
+
// Asked again by the next event instead of staying silent for the whole window.
|
|
111
|
+
undo();
|
|
112
|
+
logger.warn(
|
|
113
|
+
{ error: error instanceof Error ? error.name : "unknown" },
|
|
114
|
+
"an ops alert could not be posted",
|
|
115
|
+
);
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
#record(kind: string, code: string | undefined, suppressed: boolean): void {
|
|
121
|
+
const before = this.#kinds.get(kind);
|
|
122
|
+
const { kinds, now } = this.#options;
|
|
123
|
+
this.#kinds.set(kind, {
|
|
124
|
+
kind,
|
|
125
|
+
name: kinds && Object.hasOwn(kinds, kind) ? (kinds[kind] ?? kind) : kind,
|
|
126
|
+
at: (now ?? Date.now)(),
|
|
127
|
+
code,
|
|
128
|
+
raised: (before?.raised ?? 0) + 1,
|
|
129
|
+
suppressed: (before?.suppressed ?? 0) + (suppressed ? 1 : 0),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|