pi-extensible-workflows 0.3.2 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/src/agent-execution.d.ts +81 -8
- package/dist/src/agent-execution.js +465 -75
- package/dist/src/ambient-workflow-evals.js +14 -33
- package/dist/src/doctor.d.ts +4 -1
- package/dist/src/doctor.js +57 -14
- package/dist/src/index.d.ts +222 -24
- package/dist/src/index.js +1548 -410
- package/dist/src/persistence.d.ts +28 -1
- package/dist/src/persistence.js +126 -9
- package/dist/src/session-inspector.d.ts +13 -1
- package/dist/src/session-inspector.js +22 -4
- package/dist/src/workflow-evals.d.ts +1 -0
- package/dist/src/workflow-evals.js +23 -25
- package/package.json +5 -2
- package/skills/pi-extensible-workflows/SKILL.md +14 -6
- package/src/agent-execution.ts +391 -86
- package/src/ambient-workflow-evals.ts +18 -28
- package/src/doctor.ts +60 -16
- package/src/index.ts +1332 -395
- package/src/persistence.ts +98 -10
- package/src/session-inspector.ts +25 -7
- package/src/workflow-evals.ts +13 -16
package/src/persistence.ts
CHANGED
|
@@ -1,20 +1,30 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { execFile } from "node:child_process";
|
|
3
|
+
import { renameSync, rmSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { access, link, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
4
5
|
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
5
6
|
import { homedir } from "node:os";
|
|
6
7
|
import { promisify } from "node:util";
|
|
7
|
-
import type { JsonValue, LaunchSnapshot, RunRecord } from "./index.js";
|
|
8
|
+
import type { BudgetApprovalRequest, JsonValue, LaunchSnapshot, RunRecord, WorkflowRunEvent } from "./index.js";
|
|
8
9
|
import type { OwnershipRecord } from "./agent-execution.js";
|
|
9
10
|
import { loadLaunchSnapshot, WorkflowError } from "./index.js";
|
|
10
11
|
|
|
11
12
|
export interface NativeSessionReference { sessionId: string; sessionFile: string }
|
|
12
13
|
export interface EffectiveSystemPrompt { sessionId: string; attempt: number; turn: number; sha256: string; prompt: string }
|
|
14
|
+
export interface ConversationHead { turn: number; sessionId: string; sessionFile: string; leafId: string; systemPrompt: string; systemPromptSha256: string; toolDefinitionsSha256: string }
|
|
15
|
+
export interface PersistedConversation { id: string; policy: JsonValue; head: ConversationHead }
|
|
16
|
+
type ConversationArtifact = { version: 1; conversations: Record<string, PersistedConversation> };
|
|
17
|
+
function isConversationArtifact(value: unknown): value is ConversationArtifact {
|
|
18
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
19
|
+
const artifact = value as { version?: unknown; conversations?: unknown };
|
|
20
|
+
return artifact.version === 1 && Boolean(artifact.conversations) && typeof artifact.conversations === "object" && !Array.isArray(artifact.conversations);
|
|
21
|
+
}
|
|
13
22
|
export interface PersistedRun extends RunRecord { nativeSessions: readonly NativeSessionReference[] }
|
|
14
23
|
export interface CompletedOperation { path: string; value: JsonValue }
|
|
15
24
|
export interface AwaitingCheckpoint { path: string; name: string; prompt: string; context: JsonValue }
|
|
25
|
+
export type PendingWorkflowDecision = BudgetApprovalRequest
|
|
16
26
|
export type PersistedOwnershipNode = OwnershipRecord
|
|
17
|
-
type Journal = { completed: Record<string, CompletedOperation>; awaiting?: Record<string, AwaitingCheckpoint> };
|
|
27
|
+
type Journal = { completed: Record<string, CompletedOperation>; awaiting?: Record<string, AwaitingCheckpoint>; decisions?: Record<string, PendingWorkflowDecision> };
|
|
18
28
|
export interface WorktreeReference { owner: string; path: string; branch: string; cwd: string; base: string }
|
|
19
29
|
|
|
20
30
|
const execute = promisify(execFile);
|
|
@@ -149,10 +159,28 @@ export function structuralPath(...names: string[]): string {
|
|
|
149
159
|
return names.map((name) => encodeURIComponent(name)).join("/");
|
|
150
160
|
}
|
|
151
161
|
|
|
152
|
-
|
|
162
|
+
export function atomicWriteFile(path: string, content: string): Promise<void>;
|
|
163
|
+
export function atomicWriteFile(path: string, content: string, sync: true): void;
|
|
164
|
+
export function atomicWriteFile(path: string, content: string, sync = false): Promise<void> | void {
|
|
153
165
|
const temporary = `${path}.${String(process.pid)}.${randomUUID()}.tmp`;
|
|
154
|
-
|
|
155
|
-
|
|
166
|
+
if (sync) {
|
|
167
|
+
try {
|
|
168
|
+
writeFileSync(temporary, content, { encoding: "utf8", mode: 0o600 });
|
|
169
|
+
renameSync(temporary, path);
|
|
170
|
+
} catch (error) {
|
|
171
|
+
try { rmSync(temporary, { force: true }); } catch { /* Preserve the original write error. */ }
|
|
172
|
+
throw error;
|
|
173
|
+
}
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
return writeFile(temporary, content, { encoding: "utf8", mode: 0o600 }).then(() => rename(temporary, path)).catch(async (error: unknown) => {
|
|
177
|
+
try { await rm(temporary, { force: true }); } catch { /* Preserve the original write error. */ }
|
|
178
|
+
throw error;
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
async function atomicJson(path: string, value: unknown): Promise<void> {
|
|
183
|
+
await atomicWriteFile(path, `${JSON.stringify(value)}\n`);
|
|
156
184
|
}
|
|
157
185
|
|
|
158
186
|
async function json<T>(path: string): Promise<T> { return JSON.parse(await readFile(path, "utf8")) as T; }
|
|
@@ -164,8 +192,10 @@ export class RunStore {
|
|
|
164
192
|
private stateWrite: Promise<void> = Promise.resolve();
|
|
165
193
|
private worktreeWrite: Promise<void> = Promise.resolve();
|
|
166
194
|
private snapshotWrite: Promise<void> = Promise.resolve();
|
|
195
|
+
private launchSnapshotWrite: Promise<void> = Promise.resolve();
|
|
167
196
|
// ponytail: the session lease prevents concurrent RunStore writers for one run.
|
|
168
197
|
private systemPromptWrite: Promise<void> = Promise.resolve();
|
|
198
|
+
private conversationWrite: Promise<void> = Promise.resolve();
|
|
169
199
|
constructor(readonly cwd: string, readonly sessionId: string, readonly runId: string, home = homedir()) {
|
|
170
200
|
this.cwd = resolve(cwd);
|
|
171
201
|
this.directory = join(runsDirectory(this.cwd, sessionId, home), safePart(runId));
|
|
@@ -177,12 +207,14 @@ export class RunStore {
|
|
|
177
207
|
await mkdir(dirname(this.directory), { recursive: true, mode: 0o700 });
|
|
178
208
|
await mkdir(temporary, { mode: 0o700 });
|
|
179
209
|
try {
|
|
210
|
+
await writeFile(join(temporary, "workflow.js"), snapshot.script, { encoding: "utf8", mode: 0o600 });
|
|
180
211
|
await atomicJson(join(temporary, "snapshot.json"), snapshot);
|
|
181
|
-
await atomicJson(join(temporary, "journal.json"), { completed: {}, awaiting: {} });
|
|
212
|
+
await atomicJson(join(temporary, "journal.json"), { completed: {}, awaiting: {}, decisions: {} });
|
|
182
213
|
await atomicJson(join(temporary, "ownership.json"), []);
|
|
183
214
|
await atomicJson(join(temporary, "worktrees.json"), []);
|
|
184
215
|
await atomicJson(join(temporary, "state.json"), run);
|
|
185
216
|
await atomicJson(join(temporary, "system-prompts.json"), { version: 1, entries: [] });
|
|
217
|
+
await atomicJson(join(temporary, "conversations.json"), { version: 1, conversations: {} });
|
|
186
218
|
await rename(temporary, this.directory);
|
|
187
219
|
} catch (error) {
|
|
188
220
|
await rm(temporary, { recursive: true, force: true });
|
|
@@ -225,6 +257,16 @@ export class RunStore {
|
|
|
225
257
|
return result;
|
|
226
258
|
}
|
|
227
259
|
|
|
260
|
+
async saveSnapshot(snapshot: Readonly<LaunchSnapshot>): Promise<void> {
|
|
261
|
+
const write = this.launchSnapshotWrite.then(() => atomicJson(join(this.directory, "snapshot.json"), snapshot));
|
|
262
|
+
this.launchSnapshotWrite = write.catch(() => undefined);
|
|
263
|
+
await write;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
async appendEvent(event: WorkflowRunEvent): Promise<void> {
|
|
267
|
+
await this.updateState((run) => ({ ...run, events: [...(run.events ?? []), ...(run.events?.some((current) => current.type === event.type && current.message === event.message) ? [] : [event])] }));
|
|
268
|
+
}
|
|
269
|
+
|
|
228
270
|
async saveOwnership(nodes: readonly PersistedOwnershipNode[]): Promise<void> {
|
|
229
271
|
await atomicJson(join(this.directory, "ownership.json"), nodes);
|
|
230
272
|
}
|
|
@@ -251,6 +293,27 @@ export class RunStore {
|
|
|
251
293
|
return (await json<{ version: 1; entries: EffectiveSystemPrompt[] }>(this.systemPromptPath()).catch((error: unknown) => { if ((error as NodeJS.ErrnoException).code === "ENOENT") return { version: 1 as const, entries: [] }; throw error; })).entries;
|
|
252
294
|
}
|
|
253
295
|
|
|
296
|
+
conversationPath(): string { return join(this.directory, "conversations.json"); }
|
|
297
|
+
async conversation(id: string): Promise<PersistedConversation | undefined> {
|
|
298
|
+
await this.conversationWrite;
|
|
299
|
+
let artifact: ConversationArtifact;
|
|
300
|
+
try { const raw = await json<unknown>(this.conversationPath()); if (!isConversationArtifact(raw)) throw new WorkflowError("RESUME_INCOMPATIBLE", "Conversation state is corrupt"); artifact = raw; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; if (error instanceof WorkflowError) throw error; throw new WorkflowError("RESUME_INCOMPATIBLE", `Cannot load conversation state: ${error instanceof Error ? error.message : String(error)}`); }
|
|
301
|
+
return artifact.conversations[id];
|
|
302
|
+
}
|
|
303
|
+
async saveConversation(conversation: PersistedConversation): Promise<void> {
|
|
304
|
+
const write = this.conversationWrite.then(async () => {
|
|
305
|
+
const path = this.conversationPath();
|
|
306
|
+
let artifact: ConversationArtifact;
|
|
307
|
+
try { const raw = await json<unknown>(path); if (!isConversationArtifact(raw)) throw new WorkflowError("RESUME_INCOMPATIBLE", "Conversation state is corrupt"); artifact = raw; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") artifact = { version: 1, conversations: {} }; else if (error instanceof WorkflowError) throw error; else throw new WorkflowError("RESUME_INCOMPATIBLE", `Cannot load conversation state: ${error instanceof Error ? error.message : String(error)}`); }
|
|
308
|
+
const previous = artifact.conversations[conversation.id];
|
|
309
|
+
if (previous && previous.head.turn + 1 !== conversation.head.turn) throw new WorkflowError("RESUME_INCOMPATIBLE", `Conversation head is not the previous turn: ${conversation.id}`);
|
|
310
|
+
if (!previous && conversation.head.turn !== 1) throw new WorkflowError("RESUME_INCOMPATIBLE", `Conversation must start at turn one: ${conversation.id}`);
|
|
311
|
+
artifact.conversations[conversation.id] = structuredClone(conversation);
|
|
312
|
+
await atomicJson(path, artifact);
|
|
313
|
+
});
|
|
314
|
+
this.conversationWrite = write.catch(() => undefined);
|
|
315
|
+
await write;
|
|
316
|
+
}
|
|
254
317
|
private async updateJournal<T>(update: (journal: Journal) => T | Promise<T>): Promise<T> {
|
|
255
318
|
let result!: T;
|
|
256
319
|
const write = this.journalWrite.then(async () => {
|
|
@@ -291,13 +354,30 @@ export class RunStore {
|
|
|
291
354
|
const journal = await json<Journal>(join(this.directory, "journal.json"));
|
|
292
355
|
return Object.values(journal.awaiting ?? {});
|
|
293
356
|
}
|
|
357
|
+
async requestWorkflowDecision(request: PendingWorkflowDecision): Promise<void> {
|
|
358
|
+
await this.updateJournal((journal) => { journal.decisions ??= {}; journal.decisions[request.proposalId] = request; });
|
|
359
|
+
}
|
|
360
|
+
async pendingWorkflowDecisions(): Promise<readonly PendingWorkflowDecision[]> {
|
|
361
|
+
await this.journalWrite;
|
|
362
|
+
const journal = await json<Journal>(join(this.directory, "journal.json"));
|
|
363
|
+
return Object.values(journal.decisions ?? {});
|
|
364
|
+
}
|
|
365
|
+
async answerWorkflowDecision(proposalId: string, approved: boolean): Promise<PendingWorkflowDecision | undefined> {
|
|
366
|
+
return this.updateJournal((journal) => {
|
|
367
|
+
const request = journal.decisions?.[proposalId];
|
|
368
|
+
if (!request) return undefined;
|
|
369
|
+
journal.completed[`decision/${proposalId}`] = { path: `decision/${proposalId}`, value: approved };
|
|
370
|
+
delete journal.decisions?.[proposalId];
|
|
371
|
+
return request;
|
|
372
|
+
});
|
|
373
|
+
}
|
|
294
374
|
|
|
295
375
|
async answerCheckpoint(name: string, approved: boolean): Promise<AwaitingCheckpoint | undefined> {
|
|
296
376
|
return this.updateJournal((journal) => {
|
|
297
|
-
const checkpoint = Object.values(journal.awaiting
|
|
377
|
+
const checkpoint = Object.values(journal.awaiting ?? {}).find((item) => item.name === name);
|
|
298
378
|
if (!checkpoint || journal.completed[checkpoint.path]) return undefined;
|
|
299
379
|
journal.completed[checkpoint.path] = { path: checkpoint.path, value: approved };
|
|
300
|
-
journal.awaiting = Object.fromEntries(Object.entries(journal.awaiting
|
|
380
|
+
journal.awaiting = Object.fromEntries(Object.entries(journal.awaiting ?? {}).filter(([path]) => path !== checkpoint.path));
|
|
301
381
|
return checkpoint;
|
|
302
382
|
});
|
|
303
383
|
}
|
|
@@ -412,8 +492,16 @@ export class RunStore {
|
|
|
412
492
|
try {
|
|
413
493
|
const write = this.snapshotWrite.then(async () => {
|
|
414
494
|
const record = await this.worktree(owner);
|
|
415
|
-
|
|
416
|
-
|
|
495
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
496
|
+
await git(record.path, ["add", "-A"]);
|
|
497
|
+
if (!(await git(record.path, ["status", "--porcelain"])).trim()) break;
|
|
498
|
+
try {
|
|
499
|
+
await git(record.path, ["commit", "-m", "pi-extensible-workflows runtime snapshot"], gitIdentity);
|
|
500
|
+
break;
|
|
501
|
+
} catch (error) {
|
|
502
|
+
if (attempt === 2) throw error;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
417
505
|
return (await git(record.path, ["rev-parse", "HEAD"])).trim();
|
|
418
506
|
});
|
|
419
507
|
this.snapshotWrite = write.then(() => undefined, () => undefined);
|
package/src/session-inspector.ts
CHANGED
|
@@ -4,13 +4,13 @@ import { emitKeypressEvents } from "node:readline";
|
|
|
4
4
|
import { createInterface } from "node:readline/promises";
|
|
5
5
|
import { stdin, stdout } from "node:process";
|
|
6
6
|
import { highlightCode, initTheme, SessionManager, truncateToVisualLines, type SessionEntry, type SessionInfo } from "@earendil-works/pi-coding-agent";
|
|
7
|
-
import { inspectWorkflowScript, type ModelSpec, type StaticWorkflowCall } from "./index.js";
|
|
7
|
+
import { formatBudgetStatus, inspectWorkflowScript, type AgentSetupSummary, type ModelSpec, type StaticWorkflowCall } from "./index.js";
|
|
8
8
|
import { listRunIds, RunStore, type PersistedRun } from "./persistence.js";
|
|
9
9
|
|
|
10
10
|
export interface ModelUsage { model: string; cost: number }
|
|
11
|
-
export interface AttemptReport { attempt: number; prompt: string; model: string; thinking?: ModelSpec["thinking"]; cost: number; models: readonly ModelUsage[]; error?: string }
|
|
12
|
-
export interface AgentReport { name: string; label?: string; state: string; role?: string; model: string; thinking?: ModelSpec["thinking"]; cost: number; attempts: readonly AttemptReport[] }
|
|
13
|
-
export interface WorkflowReport { name: string; description?: string; status: string; runId?: string; script?: string; calls: readonly StaticWorkflowCall[]; parseError?: string; cost: number; models: readonly ModelUsage[]; agents: readonly AgentReport[] }
|
|
11
|
+
export interface AttemptReport { attempt: number; prompt: string; model: string; thinking?: ModelSpec["thinking"]; cost: number; models: readonly ModelUsage[]; error?: string; setup?: AgentSetupSummary }
|
|
12
|
+
export interface AgentReport { name: string; label?: string; state: string; role?: string; requestedModel?: string; model: string; thinking?: ModelSpec["thinking"]; cost: number; attempts: readonly AttemptReport[]; setup?: AgentSetupSummary }
|
|
13
|
+
export interface WorkflowReport { name: string; description?: string; status: string; runId?: string; script?: string; calls: readonly StaticWorkflowCall[]; parseError?: string; cost: number; models: readonly ModelUsage[]; agents: readonly AgentReport[]; budget?: PersistedRun["budget"]; budgetVersion?: number; usage?: PersistedRun["usage"]; budgetEvents?: PersistedRun["budgetEvents"]; events?: readonly { type: string; message: string }[] }
|
|
14
14
|
export interface SessionReport { id: string; cwd: string; path: string; cost: number; models: readonly ModelUsage[]; workflows: readonly WorkflowReport[]; totalCost: number; totalModels: readonly ModelUsage[] }
|
|
15
15
|
export interface InspectorViewState { view: "list" | "detail" | "script"; selected: number; scroll: number }
|
|
16
16
|
|
|
@@ -163,6 +163,7 @@ async function agentReport(agent: PersistedRun["agents"][number]): Promise<Agent
|
|
|
163
163
|
cost,
|
|
164
164
|
models: log.models.length ? log.models : [{ model, cost }],
|
|
165
165
|
...(attempt.error ? { error: `${attempt.error.code}: ${attempt.error.message}` } : {}),
|
|
166
|
+
...(attempt.setup ? { setup: attempt.setup } : {}),
|
|
166
167
|
});
|
|
167
168
|
continue;
|
|
168
169
|
}
|
|
@@ -175,6 +176,7 @@ async function agentReport(agent: PersistedRun["agents"][number]): Promise<Agent
|
|
|
175
176
|
cost,
|
|
176
177
|
models: [{ model: fallbackModel, cost }],
|
|
177
178
|
...(attempt.error ? { error: `${attempt.error.code}: ${attempt.error.message}` } : {}),
|
|
179
|
+
...(attempt.setup ? { setup: attempt.setup } : {}),
|
|
178
180
|
});
|
|
179
181
|
}
|
|
180
182
|
if (!attempts.length) {
|
|
@@ -182,7 +184,7 @@ async function agentReport(agent: PersistedRun["agents"][number]): Promise<Agent
|
|
|
182
184
|
attempts.push({ attempt: 1, prompt: "(transcript unavailable)", model: fallbackModel, ...(fallbackThinking !== undefined ? { thinking: fallbackThinking } : {}), cost, models: [{ model: fallbackModel, cost }] });
|
|
183
185
|
}
|
|
184
186
|
const latest = attempts[attempts.length - 1];
|
|
185
|
-
return { name: agent.name, ...(agent.label ? { label: agent.label } : {}), state: agent.state, ...(agent.role ? { role: agent.role } : {}), model: latest?.model ?? fallbackModel, ...(latest?.thinking !== undefined ? { thinking: latest.thinking } : {}), cost: attempts.reduce((sum, attempt) => sum + attempt.cost, 0), attempts };
|
|
187
|
+
return { name: agent.name, ...(agent.label ? { label: agent.label } : {}), state: agent.state, ...(agent.role ? { role: agent.role } : {}), ...(agent.requestedModel ? { requestedModel: agent.requestedModel } : {}), model: latest?.model ?? fallbackModel, ...(latest?.thinking !== undefined ? { thinking: latest.thinking } : {}), cost: attempts.reduce((sum, attempt) => sum + attempt.cost, 0), attempts, ...(latest?.setup ? { setup: latest.setup } : {}) };
|
|
186
188
|
}
|
|
187
189
|
|
|
188
190
|
export function matchSession(query: string, sessions: readonly SessionInfo[]): SessionInfo {
|
|
@@ -229,6 +231,11 @@ export async function loadSessionReport(path: string, home = homedir()): Promise
|
|
|
229
231
|
cost: agents.reduce((sum, agent) => sum + agent.cost, 0),
|
|
230
232
|
models,
|
|
231
233
|
agents,
|
|
234
|
+
...(loaded?.run.budget ? { budget: loaded.run.budget } : {}),
|
|
235
|
+
...(loaded?.run.budgetVersion !== undefined ? { budgetVersion: loaded.run.budgetVersion } : {}),
|
|
236
|
+
...(loaded?.run.usage ? { usage: loaded.run.usage } : {}),
|
|
237
|
+
...(loaded?.run.budgetEvents ? { budgetEvents: loaded.run.budgetEvents } : {}),
|
|
238
|
+
...(loaded?.run.events?.length ? { events: loaded.run.events } : {})
|
|
232
239
|
});
|
|
233
240
|
}
|
|
234
241
|
const workflowCost = workflows.reduce((sum, workflow) => sum + workflow.cost, 0);
|
|
@@ -253,7 +260,9 @@ function detailLines(workflow: WorkflowReport): string[] {
|
|
|
253
260
|
style(ansi.bold + ansi.cyan, workflow.name),
|
|
254
261
|
`${workflow.status} · ${money(workflow.cost)}${workflow.runId ? ` · ${workflow.runId}` : ""}`,
|
|
255
262
|
workflow.description ?? "",
|
|
263
|
+
...(workflow.events?.length ? ["", style(ansi.bold, "Run events"), ...workflow.events.map((event) => `${event.type}: ${event.message}`)] : []),
|
|
256
264
|
"",
|
|
265
|
+
...(workflow.budget ? formatBudgetStatus({ budget: workflow.budget, ...(workflow.budgetVersion !== undefined ? { budgetVersion: workflow.budgetVersion } : {}), ...(workflow.usage ? { usage: workflow.usage } : {}), ...(workflow.budgetEvents ? { budgetEvents: workflow.budgetEvents } : {}) }).map((line) => `Budget ${line}`) : []),
|
|
257
266
|
style(ansi.bold, "Models"),
|
|
258
267
|
modelSummary(workflow.models),
|
|
259
268
|
"",
|
|
@@ -267,9 +276,18 @@ function detailLines(workflow: WorkflowReport): string[] {
|
|
|
267
276
|
];
|
|
268
277
|
if (!workflow.agents.length) lines.push("(no agent run was persisted)");
|
|
269
278
|
for (const agent of workflow.agents) {
|
|
270
|
-
lines.push("", style(agent.state === "completed" ? ansi.green : agent.state === "failed" ? ansi.red : ansi.yellow, `${agent.label ?? agent.name} [${agent.state}]`), `${agent.role ? `role=${agent.role} · ` : ""}${agent.model}${agent.thinking !== undefined ? `:${agent.thinking}` : ""} · ${money(agent.cost)}`);
|
|
279
|
+
lines.push("", style(agent.state === "completed" ? ansi.green : agent.state === "failed" ? ansi.red : ansi.yellow, `${agent.label ?? agent.name} [${agent.state}]`), `${agent.role ? `role=${agent.role} · ` : ""}${agent.requestedModel ? `requested=${agent.requestedModel} · ` : ""}${agent.model}${agent.thinking !== undefined ? `:${agent.thinking}` : ""} · ${money(agent.cost)}`);
|
|
271
280
|
for (const attempt of agent.attempts) {
|
|
272
|
-
lines.push(`Attempt ${String(attempt.attempt)} · ${attempt.model}${attempt.thinking !== undefined ? `:${attempt.thinking}` : ""} · ${money(attempt.cost)}${attempt.error ? ` · ${attempt.error}` : ""}`, `Prompt: ${attempt.prompt}
|
|
281
|
+
lines.push(`Attempt ${String(attempt.attempt)} · ${attempt.model}${attempt.thinking !== undefined ? `:${attempt.thinking}` : ""} · ${money(attempt.cost)}${attempt.error ? ` · ${attempt.error}` : ""}`, `Prompt: ${attempt.prompt}`, ...(attempt.setup ? [
|
|
282
|
+
`Hooks: ${attempt.setup.hookNames.join(", ") || "(none)"}`,
|
|
283
|
+
`Effective: model=${attempt.setup.model.provider}/${attempt.setup.model.model}${attempt.setup.model.thinking ? `:${attempt.setup.model.thinking}` : ""} tools=${attempt.setup.tools.join(",") || "(none)"} cwd=${attempt.setup.cwd}`,
|
|
284
|
+
...(attempt.setup.disabledAgentResources ? [
|
|
285
|
+
`Disabled skills: ${attempt.setup.disabledAgentResources.skills.join(", ") || "(none)"}`,
|
|
286
|
+
`Disabled extensions: ${attempt.setup.disabledAgentResources.extensions.join(", ") || "(none)"}`,
|
|
287
|
+
`Unmatched skills: ${attempt.setup.disabledAgentResources.unmatchedSkills.join(", ") || "(none)"}`,
|
|
288
|
+
`Unmatched extensions: ${attempt.setup.disabledAgentResources.unmatchedExtensions.join(", ") || "(none)"}`,
|
|
289
|
+
] : []),
|
|
290
|
+
] : []));
|
|
273
291
|
}
|
|
274
292
|
}
|
|
275
293
|
return lines.filter((line, index) => line || index !== 2);
|
package/src/workflow-evals.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { Value } from "typebox/value";
|
|
|
8
8
|
import { getAgentDir, parseFrontmatter, SessionManager } from "@earendil-works/pi-coding-agent";
|
|
9
9
|
import { CAPTURE_ERROR_PREFIX, CAPTURE_IDENTITY, resolveWorkflowSkillPath } from "./eval-capture-extension.js";
|
|
10
10
|
export { resolveWorkflowSkillPath } from "./eval-capture-extension.js";
|
|
11
|
-
import { ERROR_CODES, inspectWorkflowScript, loadAgentDefinitions, runWorkflow, WorkflowError, type AgentIdentity, type JsonSchema, type JsonValue, type StaticWorkflowCall, type StaticWorkflowExecution, type WorkflowErrorCode } from "./index.js";
|
|
11
|
+
import { ERROR_CODES, inspectWorkflowScript, isObject, loadAgentDefinitions, runWorkflow, WORKFLOW_CALL_KINDS, WorkflowError, type AgentIdentity, type JsonSchema, type JsonValue, type StaticWorkflowCall, type StaticWorkflowExecution, type WorkflowErrorCode } from "./index.js";
|
|
12
12
|
|
|
13
13
|
export type SignificantAction = { kind: "tool"; name: string } | { kind: "text" } | { kind: "thinking" };
|
|
14
14
|
export type SequenceExpectation = readonly string[] | { equals?: readonly string[]; startsWith?: readonly string[] };
|
|
@@ -95,7 +95,6 @@ export const SAFE_PARENT_EVAL_TOOLS = Object.freeze(["read", "grep", "find", "ba
|
|
|
95
95
|
const EVAL_MODEL_TOKEN = "$EVAL_MODEL";
|
|
96
96
|
const semantic = (description: string): readonly SemanticCriterion[] => [{ id: "intent", description }];
|
|
97
97
|
const JSON_RESULT_TYPES = ["null", "boolean", "number", "integer", "string", "array", "object"] as const;
|
|
98
|
-
const WORKFLOW_CALL_KINDS = ["agent", "parallel", "pipeline", "checkpoint", "phase", "withWorktree"] as const;
|
|
99
98
|
const AGENT_OPTION_NAMES = ["role", "model", "thinking", "tools", "retries"] as const;
|
|
100
99
|
const expectationKeys = ["firstSignificantAction", "firstTool", "firstBatchToolSequence", "parentToolSequence", "workflowCallCount", "requiredOperations", "forbiddenOperations", "requiredRoles", "minimumAgentCalls", "requireOutputSchema", "expectedResults", "agentPolicies", "requiredAgentOrder", "requiredAgentStructures", "requiredDataFlow"] as const;
|
|
101
100
|
const caseKeys = ["id", "prompt", "timeoutMs", "maxCost", "expectations", "expectedWorkflowCalls", "semanticCriteria"] as const;
|
|
@@ -130,7 +129,6 @@ export function loadWorkflowEvalCases(directory = evalCasesDirectory()): readonl
|
|
|
130
129
|
export const INITIAL_WORKFLOW_EVAL_CASES: readonly WorkflowEvalCase[] = loadWorkflowEvalCases();
|
|
131
130
|
|
|
132
131
|
|
|
133
|
-
function isObject(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
|
|
134
132
|
function isJson(value: unknown): value is JsonValue { if (value === null || typeof value === "string" || typeof value === "boolean") return true; if (typeof value === "number") return Number.isFinite(value); if (Array.isArray(value)) return value.every(isJson); return isObject(value) && Object.values(value).every(isJson); }
|
|
135
133
|
function asJsonObject(value: unknown): Readonly<Record<string, JsonValue>> | undefined { return isObject(value) && Object.values(value).every(isJson) ? value as Readonly<Record<string, JsonValue>> : undefined; }
|
|
136
134
|
function jsonType(value: JsonValue): JsonResultType { if (value === null) return "null"; if (Array.isArray(value)) return "array"; if (typeof value === "number") return Number.isInteger(value) ? "integer" : "number"; if (typeof value === "object") return "object"; if (typeof value === "string") return "string"; if (typeof value === "boolean") return "boolean"; return "null"; }
|
|
@@ -646,21 +644,20 @@ function seedEvalProject(cwd: string, home: string, model: string): void {
|
|
|
646
644
|
if (frontmatterEnd >= 0) writeFileSync(path, `${content.slice(0, frontmatterEnd).replace(/^model:.*$/m, `model: ${model}`)}${content.slice(frontmatterEnd)}`);
|
|
647
645
|
}
|
|
648
646
|
}
|
|
649
|
-
|
|
647
|
+
export function findSessionFile(directory: string, sessionId: string): string | undefined {
|
|
648
|
+
if (!existsSync(directory)) return undefined;
|
|
649
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
650
|
+
const path = join(directory, entry.name);
|
|
651
|
+
if (entry.isDirectory()) { const found = findSessionFile(path, sessionId); if (found) return found; }
|
|
652
|
+
else if (entry.name.endsWith(".jsonl")) {
|
|
653
|
+
try { const header = JSON.parse(readFileSync(path, "utf8").split("\n")[0] ?? "{}") as unknown; if (isObject(header) && header.id === sessionId) return path; } catch { /* Ignore incomplete sessions. */ }
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
return undefined;
|
|
657
|
+
}
|
|
650
658
|
async function findParentSession(cwd: string, sessionDir: string, sessionId: string): Promise<string | undefined> {
|
|
651
659
|
try { const sessions = await SessionManager.list(cwd, sessionDir); const found = sessions.find((session) => session.id === sessionId); if (found) return found.path; } catch { /* Fall through to the JSONL scan. */ }
|
|
652
|
-
|
|
653
|
-
if (!existsSync(directory)) return undefined;
|
|
654
|
-
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
655
|
-
const path = join(directory, entry.name);
|
|
656
|
-
if (entry.isDirectory()) { const found = visit(path); if (found) return found; }
|
|
657
|
-
else if (entry.name.endsWith(".jsonl")) {
|
|
658
|
-
try { const header = JSON.parse(readFileSync(path, "utf8").split("\n")[0] ?? "{}") as unknown; if (isObject(header) && header.id === sessionId) return path; } catch { /* Ignore incomplete files. */ }
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
return undefined;
|
|
662
|
-
};
|
|
663
|
-
return visit(sessionDir);
|
|
660
|
+
return findSessionFile(sessionDir, sessionId);
|
|
664
661
|
}
|
|
665
662
|
|
|
666
663
|
function emptyAccounting(cost = 0): EvalAccounting { return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost, models: [] }; }
|