pi-extensible-workflows 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -0
- package/dist/src/agent-execution.d.ts +213 -0
- package/dist/src/agent-execution.js +537 -0
- package/dist/src/ambient-workflow-evals.d.ts +112 -0
- package/dist/src/ambient-workflow-evals.js +375 -0
- package/dist/src/cli.d.ts +6 -0
- package/dist/src/cli.js +26 -0
- package/dist/src/doctor.d.ts +59 -0
- package/dist/src/doctor.js +269 -0
- package/dist/src/eval-capture-extension.d.ts +5 -0
- package/dist/src/eval-capture-extension.js +48 -0
- package/dist/src/index.d.ts +362 -0
- package/dist/src/index.js +2839 -0
- package/dist/src/persistence.d.ts +90 -0
- package/dist/src/persistence.js +530 -0
- package/dist/src/session-inspector.d.ts +59 -0
- package/dist/src/session-inspector.js +396 -0
- package/dist/src/workflow-evals-child.d.ts +1 -0
- package/dist/src/workflow-evals-child.js +13 -0
- package/dist/src/workflow-evals.d.ts +290 -0
- package/dist/src/workflow-evals.js +1221 -0
- package/evals/cases/custom-model-read.yaml +15 -0
- package/evals/cases/direct-answer.yaml +6 -0
- package/evals/cases/mixed-parallel-pipeline.yaml +18 -0
- package/evals/cases/output-schema.yaml +22 -0
- package/evals/cases/parallel.yaml +15 -0
- package/evals/cases/pipeline.yaml +10 -0
- package/evals/cases/ready-for-agent-parallel-merge.yaml +32 -0
- package/evals/cases/required-role.yaml +14 -0
- package/evals/cases/role-model-mixed.yaml +18 -0
- package/evals/cases/two-agents.yaml +10 -0
- package/package.json +65 -0
- package/skills/pi-extensible-workflows/SKILL.md +109 -0
- package/src/agent-execution.ts +529 -0
- package/src/ambient-workflow-evals.ts +452 -0
- package/src/cli.ts +23 -0
- package/src/doctor.ts +285 -0
- package/src/eval-capture-extension.ts +54 -0
- package/src/index.ts +2519 -0
- package/src/persistence.ts +470 -0
- package/src/session-inspector.ts +370 -0
- package/src/workflow-evals-child.ts +15 -0
- package/src/workflow-evals.ts +876 -0
- package/test/fixtures/ready-for-agent-tasks.md +9 -0
- package/test/fixtures/workflow-eval-roles/developer.md +18 -0
- package/test/fixtures/workflow-eval-roles/reviewer.md +18 -0
- package/test/fixtures/workflow-eval-roles/scout.md +17 -0
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { Type } from "@earendil-works/pi-ai";
|
|
3
|
+
import { Value } from "typebox/value";
|
|
4
|
+
import { createAgentSession, DefaultResourceLoader, getAgentDir, ModelRuntime, SessionManager } from "@earendil-works/pi-coding-agent";
|
|
5
|
+
import { parseModelReference, WorkflowError } from "./index.js";
|
|
6
|
+
function parseModel(value, fallback, thinking) {
|
|
7
|
+
if (!value)
|
|
8
|
+
return { ...fallback, ...(thinking ? { thinking } : {}) };
|
|
9
|
+
const parsed = parseModelReference(value);
|
|
10
|
+
return { ...parsed, ...(thinking ? { thinking } : !parsed.thinking && fallback.thinking ? { thinking: fallback.thinking } : {}) };
|
|
11
|
+
}
|
|
12
|
+
function modelCapability(model) { return `${model.provider}/${model.model}`; }
|
|
13
|
+
function text(messages) {
|
|
14
|
+
const message = [...messages].reverse().find((item) => item.role === "assistant");
|
|
15
|
+
if (!message || !Array.isArray(message.content))
|
|
16
|
+
return "";
|
|
17
|
+
return message.content.filter((part) => typeof part === "object" && part !== null && "type" in part && part.type === "text" && "text" in part && typeof part.text === "string").map((part) => part.text).join("");
|
|
18
|
+
}
|
|
19
|
+
function accounting(messages) {
|
|
20
|
+
const total = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 };
|
|
21
|
+
for (const message of messages)
|
|
22
|
+
if (message.role === "assistant" && message.usage) {
|
|
23
|
+
total.input += message.usage.input;
|
|
24
|
+
total.output += message.usage.output;
|
|
25
|
+
total.cacheRead += message.usage.cacheRead;
|
|
26
|
+
total.cacheWrite += message.usage.cacheWrite;
|
|
27
|
+
total.cost += message.usage.cost.total;
|
|
28
|
+
}
|
|
29
|
+
return total;
|
|
30
|
+
}
|
|
31
|
+
export async function createNativeAgentSession(input) {
|
|
32
|
+
const agentDir = input.agentDir ?? getAgentDir();
|
|
33
|
+
const manager = input.agentDir ? SessionManager.create(input.cwd, join(agentDir, "sessions")) : SessionManager.create(input.cwd);
|
|
34
|
+
manager.appendSessionInfo(input.sessionLabel);
|
|
35
|
+
const modelRuntime = await ModelRuntime.create({ authPath: join(agentDir, "auth.json"), modelsPath: join(agentDir, "models.json") });
|
|
36
|
+
const model = modelRuntime.getModel(input.model.provider, input.model.model);
|
|
37
|
+
if (!model)
|
|
38
|
+
throw new WorkflowError("UNKNOWN_MODEL", `Unknown model: ${input.model.provider}/${input.model.model}`);
|
|
39
|
+
const customTools = [...(input.customTools ?? []), ...(input.resultTool ? [input.resultTool] : [])];
|
|
40
|
+
const tools = [...new Set([...input.tools, ...customTools.map(({ name }) => name)])];
|
|
41
|
+
const resourceLoader = input.systemPromptAppend ? new DefaultResourceLoader({ cwd: input.cwd, agentDir, appendSystemPromptOverride: (base) => [...base, input.systemPromptAppend ?? ""] }) : undefined;
|
|
42
|
+
if (resourceLoader)
|
|
43
|
+
await resourceLoader.reload();
|
|
44
|
+
const { session } = await createAgentSession({ cwd: input.cwd, agentDir, modelRuntime, model, ...(input.model.thinking ? { thinkingLevel: input.model.thinking } : {}), tools, ...(customTools.length ? { customTools } : {}), ...(resourceLoader ? { resourceLoader } : {}), sessionManager: manager });
|
|
45
|
+
return session;
|
|
46
|
+
}
|
|
47
|
+
export class WorkflowAgentExecutor {
|
|
48
|
+
root;
|
|
49
|
+
createSession;
|
|
50
|
+
constructor(root, createSession = createNativeAgentSession) {
|
|
51
|
+
this.root = root;
|
|
52
|
+
this.createSession = createSession;
|
|
53
|
+
}
|
|
54
|
+
resolve(options, inheritedTools) {
|
|
55
|
+
const role = options.role;
|
|
56
|
+
const definition = role ? this.root.agentDefinitions?.[role] : undefined;
|
|
57
|
+
if (role && !definition)
|
|
58
|
+
throw new WorkflowError("UNKNOWN_AGENT_TYPE", `Unknown agent role: ${role}`);
|
|
59
|
+
if (role && (options.model !== undefined || options.thinking !== undefined || options.tools !== undefined))
|
|
60
|
+
throw new WorkflowError("INVALID_METADATA", "Role agents must not specify model, thinking, or tools");
|
|
61
|
+
const requested = options.tools !== undefined ? options.tools : definition?.tools !== undefined ? definition.tools : options.effectiveTools !== undefined ? options.effectiveTools : inheritedTools !== undefined ? inheritedTools : [...this.root.tools];
|
|
62
|
+
const forbidden = requested.find((tool) => !this.root.tools.has(tool));
|
|
63
|
+
if (forbidden)
|
|
64
|
+
throw new WorkflowError("UNKNOWN_TOOL", `Tool is outside the launching session boundary: ${forbidden}`);
|
|
65
|
+
const model = parseModel(options.model ?? definition?.model, this.root.model, options.thinking ?? definition?.thinking);
|
|
66
|
+
const availableModels = this.root.availableModels ?? new Set([modelCapability(this.root.model)]);
|
|
67
|
+
if (!availableModels.has(modelCapability(model)))
|
|
68
|
+
throw new WorkflowError("UNKNOWN_MODEL", `Unknown model: ${modelCapability(model)}`);
|
|
69
|
+
return { model, tools: [...requested], systemPromptAppend: definition?.prompt ?? "" };
|
|
70
|
+
}
|
|
71
|
+
async execute(task, options, signal, customTools = [], setSteer, beforeRetry) {
|
|
72
|
+
if (!Number.isInteger(options.retries ?? 0) || (options.retries ?? 0) < 0)
|
|
73
|
+
throw new WorkflowError("INVALID_METADATA", "retries must be a non-negative integer");
|
|
74
|
+
if (options.timeoutMs !== undefined && options.timeoutMs !== null && (!Number.isInteger(options.timeoutMs) || options.timeoutMs <= 0))
|
|
75
|
+
throw new WorkflowError("INVALID_METADATA", "timeoutMs must be null or a positive integer");
|
|
76
|
+
const resolved = this.resolve(options);
|
|
77
|
+
let cwd;
|
|
78
|
+
if (options.parent) {
|
|
79
|
+
if (!options.cwd)
|
|
80
|
+
throw new WorkflowError("INVALID_METADATA", "Child agents require their parent cwd");
|
|
81
|
+
if (options.worktreeOwner) {
|
|
82
|
+
if (!this.root.runStore)
|
|
83
|
+
throw new WorkflowError("WORKTREE_FAILED", "Worktree inheritance requires a persisted run");
|
|
84
|
+
cwd = (await this.root.runStore.validateWorktree(options.worktreeOwner, options.cwd)).cwd;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
if (options.cwd !== this.root.cwd)
|
|
88
|
+
throw new WorkflowError("INVALID_METADATA", "Shared-tree children must inherit the root cwd");
|
|
89
|
+
cwd = this.root.cwd;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
else if (options.worktreeOwner) {
|
|
93
|
+
if (!this.root.runStore)
|
|
94
|
+
throw new WorkflowError("WORKTREE_FAILED", "Worktree scope requires a persisted run");
|
|
95
|
+
const worktree = await this.root.runStore.worktree(options.worktreeOwner);
|
|
96
|
+
if (options.cwd && resolvePath(options.cwd) !== resolvePath(worktree.cwd))
|
|
97
|
+
throw new WorkflowError("WORKTREE_FAILED", "Agent cwd does not match its owned worktree");
|
|
98
|
+
cwd = worktree.cwd;
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
if (options.cwd)
|
|
102
|
+
throw new WorkflowError("INVALID_METADATA", "Only child agents or worktree scopes may provide a cwd");
|
|
103
|
+
cwd = this.root.cwd;
|
|
104
|
+
}
|
|
105
|
+
const attempts = [];
|
|
106
|
+
const maxAttempts = (options.retries ?? 0) + 1;
|
|
107
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
108
|
+
const started = Date.now();
|
|
109
|
+
let accepted = false;
|
|
110
|
+
let schemaResult;
|
|
111
|
+
const hasSchemaResult = () => schemaResult !== undefined;
|
|
112
|
+
const resultTool = options.schema ? {
|
|
113
|
+
name: "workflow_result", label: "Workflow Result", description: "Submit the terminal structured workflow result", parameters: Type.Unsafe(options.schema),
|
|
114
|
+
async execute(_id, value) {
|
|
115
|
+
if (!accepted)
|
|
116
|
+
return { content: [{ type: "text", text: "Result acceptance is not enabled yet." }], details: {}, isError: true };
|
|
117
|
+
if (!Value.Check(options.schema, value))
|
|
118
|
+
return { content: [{ type: "text", text: "Result does not match the required schema." }], details: {}, isError: true };
|
|
119
|
+
schemaResult = structuredClone(value);
|
|
120
|
+
void session?.abort?.();
|
|
121
|
+
return { content: [{ type: "text", text: "Result accepted." }], details: {} };
|
|
122
|
+
},
|
|
123
|
+
} : undefined;
|
|
124
|
+
let session;
|
|
125
|
+
const toolCalls = new Map();
|
|
126
|
+
let activity;
|
|
127
|
+
let progress = Promise.resolve();
|
|
128
|
+
let unsubscribe;
|
|
129
|
+
let systemPromptTurn = 0;
|
|
130
|
+
let systemPromptWrite = Promise.resolve();
|
|
131
|
+
let systemPromptWriteError;
|
|
132
|
+
const flushSystemPrompts = async () => {
|
|
133
|
+
await systemPromptWrite;
|
|
134
|
+
if (systemPromptWriteError)
|
|
135
|
+
throw new WorkflowError("INTERNAL_ERROR", `Failed to persist effective system prompt: ${systemPromptWriteError instanceof Error ? systemPromptWriteError.message : typeof systemPromptWriteError === "string" ? systemPromptWriteError : "unknown error"}`);
|
|
136
|
+
};
|
|
137
|
+
const report = (persist) => {
|
|
138
|
+
if (!session || !options.onProgress)
|
|
139
|
+
return;
|
|
140
|
+
const update = { accounting: accounting(session.messages), toolCalls: [...toolCalls.values()], ...(activity ? { activity } : {}), persist };
|
|
141
|
+
progress = progress.then(() => options.onProgress?.(update)).then(() => undefined);
|
|
142
|
+
};
|
|
143
|
+
try {
|
|
144
|
+
session = await this.createSession({ cwd, model: resolved.model, tools: resolved.tools, sessionLabel: `${options.workflowName}:${options.label}:attempt-${String(attempt)}`, ...(this.root.agentDir ? { agentDir: this.root.agentDir } : {}), ...(customTools.length ? { customTools } : {}), ...(resultTool ? { resultTool } : {}), ...(resolved.systemPromptAppend ? { systemPromptAppend: resolved.systemPromptAppend } : {}) });
|
|
145
|
+
await options.onAttempt?.({ attempt, sessionId: session.sessionId, sessionFile: requiredFile(session.sessionFile) });
|
|
146
|
+
unsubscribe = session.subscribe?.((event) => {
|
|
147
|
+
if (event.type === "agent_start" && session?.systemPrompt !== undefined && this.root.runStore) {
|
|
148
|
+
systemPromptTurn += 1;
|
|
149
|
+
const entry = { sessionId: session.sessionId, attempt, turn: systemPromptTurn, prompt: session.systemPrompt };
|
|
150
|
+
systemPromptWrite = systemPromptWrite.then(() => this.root.runStore?.recordSystemPrompt(entry)).then(() => undefined).catch((error) => { systemPromptWriteError ??= error; });
|
|
151
|
+
}
|
|
152
|
+
if (event.type === "message_start" && event.message.role === "assistant") {
|
|
153
|
+
activity = { kind: "text", text: "responding" };
|
|
154
|
+
report(false);
|
|
155
|
+
}
|
|
156
|
+
if (event.type === "message_end") {
|
|
157
|
+
activity = undefined;
|
|
158
|
+
if (event.message.role === "assistant")
|
|
159
|
+
report(true);
|
|
160
|
+
}
|
|
161
|
+
if (event.type === "tool_execution_start") {
|
|
162
|
+
toolCalls.set(event.toolCallId, { id: event.toolCallId, name: event.toolName, state: "running" });
|
|
163
|
+
activity = { kind: "tool", text: event.toolName };
|
|
164
|
+
report(false);
|
|
165
|
+
}
|
|
166
|
+
if (event.type === "tool_execution_end") {
|
|
167
|
+
toolCalls.set(event.toolCallId, { id: event.toolCallId, name: event.toolName, state: event.isError ? "failed" : "completed" });
|
|
168
|
+
if (activity?.kind === "tool" && activity.text === event.toolName)
|
|
169
|
+
activity = undefined;
|
|
170
|
+
report(false);
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
report(false);
|
|
174
|
+
if (setSteer) {
|
|
175
|
+
if (!session.steer)
|
|
176
|
+
throw new WorkflowError("INTERNAL_ERROR", "Native Pi session does not support steering");
|
|
177
|
+
setSteer((message) => session?.steer?.(message));
|
|
178
|
+
}
|
|
179
|
+
const context = [`Workflow: ${options.workflowName}`, `Agent: ${options.label}`, options.phase ? `Phase: ${options.phase}` : "", options.parent ? `Parent: ${options.parent}` : "", "You own this task and any direct child agents you create. Return child results to your parent; do not leave descendants running.", attempt > 1 ? `Retry attempt ${String(attempt)}. Previous state: ${options.retryState ?? attempts.at(-1)?.error?.message ?? "failed attempt"}` : ""].filter(Boolean).join("\n");
|
|
180
|
+
await promptWithProviderPause(session, `${context}\n\nTask:\n${task}`, remaining(options.timeoutMs, started), signal, this.root.providerPause);
|
|
181
|
+
if (options.schema) {
|
|
182
|
+
accepted = true;
|
|
183
|
+
try {
|
|
184
|
+
await promptWithProviderPause(session, "Submit the final result now by calling workflow_result exactly once. Do not return prose.", remaining(options.timeoutMs, started), signal, this.root.providerPause);
|
|
185
|
+
}
|
|
186
|
+
catch (error) {
|
|
187
|
+
if (!hasSchemaResult())
|
|
188
|
+
throw error;
|
|
189
|
+
}
|
|
190
|
+
if (!hasSchemaResult()) {
|
|
191
|
+
try {
|
|
192
|
+
await promptWithProviderPause(session, "Your result was missing or invalid. Repair it by calling workflow_result exactly once with a schema-valid value.", remaining(options.timeoutMs, started), signal, this.root.providerPause);
|
|
193
|
+
}
|
|
194
|
+
catch (error) {
|
|
195
|
+
if (!hasSchemaResult())
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
if (schemaResult === undefined)
|
|
200
|
+
throw new WorkflowError("RESULT_INVALID", "Agent did not submit a valid workflow_result after one repair");
|
|
201
|
+
}
|
|
202
|
+
const value = options.schema ? schemaResult : text(session.messages);
|
|
203
|
+
if (options.worktreeOwner)
|
|
204
|
+
await this.root.runStore?.snapshotWorktree(options.worktreeOwner);
|
|
205
|
+
report(true);
|
|
206
|
+
await progress;
|
|
207
|
+
await flushSystemPrompts();
|
|
208
|
+
unsubscribe?.();
|
|
209
|
+
const attemptAccounting = accounting(session.messages);
|
|
210
|
+
attempts.push({ attempt, sessionId: session.sessionId, sessionFile: requiredFile(session.sessionFile), result: value, accounting: attemptAccounting });
|
|
211
|
+
session.dispose();
|
|
212
|
+
return { value, attempts, cwd };
|
|
213
|
+
}
|
|
214
|
+
catch (error) {
|
|
215
|
+
const typed = error instanceof WorkflowError ? error : new WorkflowError("AGENT_FAILED", error instanceof Error ? error.message : String(error));
|
|
216
|
+
if (session) {
|
|
217
|
+
report(true);
|
|
218
|
+
await progress;
|
|
219
|
+
try {
|
|
220
|
+
await flushSystemPrompts();
|
|
221
|
+
}
|
|
222
|
+
catch { /* Preserve the agent failure that prompted this cleanup. */ }
|
|
223
|
+
unsubscribe?.();
|
|
224
|
+
const attemptAccounting = accounting(session.messages);
|
|
225
|
+
attempts.push({ attempt, sessionId: session.sessionId, sessionFile: requiredFile(session.sessionFile), error: { code: typed.code, message: typed.message }, accounting: attemptAccounting });
|
|
226
|
+
session.dispose();
|
|
227
|
+
}
|
|
228
|
+
if (options.worktreeOwner && typed.code !== "WORKTREE_FAILED")
|
|
229
|
+
await this.root.runStore?.snapshotWorktree(options.worktreeOwner).catch(() => undefined);
|
|
230
|
+
if (attempt === maxAttempts || typed.code === "CANCELLED" || typed.code === "WORKTREE_FAILED")
|
|
231
|
+
throw Object.assign(typed, { attempts });
|
|
232
|
+
beforeRetry?.();
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
throw new WorkflowError("AGENT_FAILED", "Agent execution failed");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
export class FairAgentScheduler {
|
|
239
|
+
runner;
|
|
240
|
+
sessionLimit;
|
|
241
|
+
writeOwnership;
|
|
242
|
+
#runs = new Map();
|
|
243
|
+
#nodes = new Map();
|
|
244
|
+
#runOrder = [];
|
|
245
|
+
#cursor = 0;
|
|
246
|
+
#active = 0;
|
|
247
|
+
#nextId = 0;
|
|
248
|
+
#persistence = Promise.resolve();
|
|
249
|
+
constructor(runner, sessionLimit = 16, writeOwnership) {
|
|
250
|
+
this.runner = runner;
|
|
251
|
+
this.sessionLimit = sessionLimit;
|
|
252
|
+
this.writeOwnership = writeOwnership;
|
|
253
|
+
if (!Number.isInteger(sessionLimit) || sessionLimit < 1 || sessionLimit > 16)
|
|
254
|
+
throw new WorkflowError("INVALID_SETTINGS", "Session concurrency must be an integer from 1 to 16");
|
|
255
|
+
}
|
|
256
|
+
addRun(runId, limit = 8, maxAgentLaunches = 1000) {
|
|
257
|
+
if (this.#runs.has(runId))
|
|
258
|
+
throw new WorkflowError("DUPLICATE_NAME", `Scheduler run already exists: ${runId}`);
|
|
259
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > this.sessionLimit || !Number.isInteger(maxAgentLaunches) || maxAgentLaunches < 1)
|
|
260
|
+
throw new WorkflowError("INVALID_SETTINGS", "Invalid run concurrency or maxAgentLaunches");
|
|
261
|
+
this.#runs.set(runId, { limit, maxAgentLaunches, logical: 0, active: 0, queue: [] });
|
|
262
|
+
this.#runOrder.push(runId);
|
|
263
|
+
}
|
|
264
|
+
spawn(runId, prompt, options, parentId) {
|
|
265
|
+
const run = this.#runs.get(runId);
|
|
266
|
+
if (!run)
|
|
267
|
+
throw new WorkflowError("INTERNAL_ERROR", `Unknown scheduler run: ${runId}`);
|
|
268
|
+
const parent = parentId ? this.#nodes.get(parentId) : undefined;
|
|
269
|
+
if (parentId && (!parent || parent.runId !== runId))
|
|
270
|
+
throw new WorkflowError("UNKNOWN_AGENT_TYPE", "Parent agent is not owned by this run");
|
|
271
|
+
const effective = this.#inherit(parent, options);
|
|
272
|
+
if (++run.logical > run.maxAgentLaunches) {
|
|
273
|
+
run.logical -= 1;
|
|
274
|
+
throw new WorkflowError("RUN_LIMIT_EXCEEDED", `Run ${runId} exceeded maxAgentLaunches`);
|
|
275
|
+
}
|
|
276
|
+
const id = `${runId}:${String(++this.#nextId)}`;
|
|
277
|
+
let resolveResult = () => undefined;
|
|
278
|
+
const promise = new Promise((resolve) => { resolveResult = resolve; });
|
|
279
|
+
const node = { id, runId, ...(parentId ? { parentId } : {}), options: effective, children: new Set(), collected: false, state: "queued", controller: new AbortController(), promise, resolve: resolveResult, task: async () => undefined, restored: false };
|
|
280
|
+
node.task = async () => {
|
|
281
|
+
if (node.controller.signal.aborted) {
|
|
282
|
+
this.#release(node.runId);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
node.state = "running";
|
|
286
|
+
this.#persist(runId);
|
|
287
|
+
try {
|
|
288
|
+
const value = await this.runner({ id, runId, ...(parentId ? { parentId } : {}), prompt, options: effective, signal: node.controller.signal, setSteer: (handler) => { node.steer = handler; } });
|
|
289
|
+
this.#settle(node, { id, ok: true, value });
|
|
290
|
+
}
|
|
291
|
+
catch (error) {
|
|
292
|
+
const typed = error instanceof WorkflowError ? error : new WorkflowError("AGENT_FAILED", error instanceof Error ? error.message : String(error));
|
|
293
|
+
this.#settle(node, { id, ok: false, error: { code: typed.code, message: typed.message } });
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
this.#nodes.set(id, node);
|
|
297
|
+
parent?.children.add(id);
|
|
298
|
+
this.#persist(runId);
|
|
299
|
+
this.#enqueue(runId, () => { void node.task(); });
|
|
300
|
+
return { id, result: promise };
|
|
301
|
+
}
|
|
302
|
+
async result(parentId, childId) {
|
|
303
|
+
const parent = this.#node(parentId);
|
|
304
|
+
const child = this.#node(childId);
|
|
305
|
+
if (child.parentId !== parentId)
|
|
306
|
+
throw new WorkflowError("UNKNOWN_AGENT_TYPE", "Results are scoped to direct children");
|
|
307
|
+
child.collected = true;
|
|
308
|
+
parent.state = "waiting_for_child";
|
|
309
|
+
this.#persist(parent.runId);
|
|
310
|
+
this.#release(parent.runId);
|
|
311
|
+
const outcome = await child.promise;
|
|
312
|
+
await new Promise((resolve) => { this.#enqueue(parent.runId, () => { resolve(); }); });
|
|
313
|
+
parent.state = "running";
|
|
314
|
+
if (parent.controller.signal.aborted)
|
|
315
|
+
throw new WorkflowError("CANCELLED", "Parent agent cancelled");
|
|
316
|
+
this.#persist(parent.runId);
|
|
317
|
+
return outcome;
|
|
318
|
+
}
|
|
319
|
+
async steer(parentId, childId, message) {
|
|
320
|
+
const child = this.#node(childId);
|
|
321
|
+
if (child.parentId !== parentId)
|
|
322
|
+
throw new WorkflowError("UNKNOWN_AGENT_TYPE", "Steering is scoped to direct children");
|
|
323
|
+
if (child.state !== "running" && child.state !== "waiting_for_child")
|
|
324
|
+
throw new WorkflowError("AGENT_FAILED", "Child is not running");
|
|
325
|
+
if (!child.steer)
|
|
326
|
+
throw new WorkflowError("AGENT_FAILED", "Child has not registered a steering handler");
|
|
327
|
+
await child.steer(message);
|
|
328
|
+
}
|
|
329
|
+
cancel(id) { this.#cancelTree(this.#node(id)); }
|
|
330
|
+
cancelChildren(id) {
|
|
331
|
+
for (const childId of this.#node(id).children) {
|
|
332
|
+
const child = this.#nodes.get(childId);
|
|
333
|
+
if (child)
|
|
334
|
+
this.#cancelTree(child);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
async cancelRun(runId) {
|
|
338
|
+
const run = this.#runs.get(runId);
|
|
339
|
+
if (!run)
|
|
340
|
+
throw new WorkflowError("INTERNAL_ERROR", `Unknown scheduler run: ${runId}`);
|
|
341
|
+
const nodes = [...this.#nodes.values()].filter((node) => node.runId === runId);
|
|
342
|
+
for (const node of nodes)
|
|
343
|
+
if (!node.parentId)
|
|
344
|
+
this.#cancelTree(node);
|
|
345
|
+
await Promise.all(nodes.map(({ promise }) => promise));
|
|
346
|
+
if (nodes.every(({ restored }) => restored))
|
|
347
|
+
run.logical = 0;
|
|
348
|
+
}
|
|
349
|
+
/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-argument, @typescript-eslint/restrict-template-expressions */
|
|
350
|
+
toolsFor(parentId, resolveTools) {
|
|
351
|
+
const parent = this.#node(parentId);
|
|
352
|
+
if (!parent.options.tools.includes("agent"))
|
|
353
|
+
return [];
|
|
354
|
+
const agentTool = {
|
|
355
|
+
name: "agent", label: "Child Agent", description: "Start a direct child agent",
|
|
356
|
+
parameters: Type.Object({ prompt: Type.String(), label: Type.String(), tools: Type.Optional(Type.Array(Type.String())), model: Type.Optional(Type.String()), thinking: Type.Optional(Type.String()), role: Type.Optional(Type.String()), outputSchema: Type.Optional(Type.Unsafe({})), retries: Type.Optional(Type.Integer({ minimum: 0 })), timeoutMs: Type.Optional(Type.Union([Type.Integer({ minimum: 1 }), Type.Null()])) }),
|
|
357
|
+
execute: async (_id, params) => {
|
|
358
|
+
if (params.role !== undefined && (params.model !== undefined || params.thinking !== undefined || params.tools !== undefined))
|
|
359
|
+
throw new WorkflowError("INVALID_METADATA", "Role agents must not specify model, thinking, or tools");
|
|
360
|
+
const tools = (params.tools !== undefined || params.role !== undefined ? resolveTools?.(params.role, params.tools, params.model, parent.options.tools, params.thinking) : undefined) ?? params.tools ?? parent.options.tools;
|
|
361
|
+
const options = { label: params.label, requestedLabel: params.label, cwd: parent.options.cwd, tools, ...(params.model ? { model: params.model } : {}), ...(params.thinking ? { thinking: params.thinking } : {}), ...(params.role ? { role: params.role } : {}), ...(params.outputSchema ? { schema: params.outputSchema } : {}), ...(params.retries === undefined ? {} : { retries: params.retries }), ...(params.timeoutMs === undefined ? {} : { timeoutMs: params.timeoutMs }) };
|
|
362
|
+
const child = this.spawn(parent.runId, params.prompt, options, parentId);
|
|
363
|
+
return { content: [{ type: "text", text: JSON.stringify({ id: child.id }) }], details: { id: child.id } };
|
|
364
|
+
},
|
|
365
|
+
};
|
|
366
|
+
const resultTool = {
|
|
367
|
+
name: "get_subagent_result", label: "Child Result", description: "Wait for a direct child and return its result",
|
|
368
|
+
parameters: Type.Object({ id: Type.String() }),
|
|
369
|
+
execute: async (_id, params) => { const value = await this.result(parentId, params.id); return { content: [{ type: "text", text: JSON.stringify(value) }], details: value }; },
|
|
370
|
+
};
|
|
371
|
+
const steerTool = {
|
|
372
|
+
name: "steer_subagent", label: "Steer Child", description: "Steer a running direct child",
|
|
373
|
+
parameters: Type.Object({ id: Type.String(), message: Type.String() }),
|
|
374
|
+
execute: async (_id, params) => { await this.steer(parentId, params.id, params.message); return { content: [{ type: "text", text: "Steering delivered." }], details: {} }; },
|
|
375
|
+
};
|
|
376
|
+
return [agentTool, resultTool, steerTool];
|
|
377
|
+
}
|
|
378
|
+
snapshot() {
|
|
379
|
+
return [...this.#nodes.values()].map(({ id, parentId, options, state }) => ({ id, ...(parentId ? { parentId } : {}), label: options.label, state, options }));
|
|
380
|
+
}
|
|
381
|
+
restoreRun(runId, limit, maxAgentLaunches, ownership) {
|
|
382
|
+
this.addRun(runId, limit, maxAgentLaunches);
|
|
383
|
+
const run = this.#runs.get(runId);
|
|
384
|
+
for (const record of ownership) {
|
|
385
|
+
if (record.id.split(":").slice(0, -1).join(":") !== runId)
|
|
386
|
+
throw new WorkflowError("RESUME_INCOMPATIBLE", `Persisted agent belongs to another run: ${record.id}`);
|
|
387
|
+
let resolveResult = () => undefined;
|
|
388
|
+
const promise = new Promise((resolve) => { resolveResult = resolve; });
|
|
389
|
+
const node = { id: record.id, runId, ...(record.parentId ? { parentId: record.parentId } : {}), options: this.#inherit(undefined, record.options), children: new Set(), collected: false, state: record.state, controller: new AbortController(), promise, resolve: resolveResult, task: async () => undefined, restored: true };
|
|
390
|
+
this.#nodes.set(node.id, node);
|
|
391
|
+
run.logical += 1;
|
|
392
|
+
this.#nextId = Math.max(this.#nextId, Number(node.id.slice(node.id.lastIndexOf(":") + 1)) || 0);
|
|
393
|
+
if (record.state === "completed")
|
|
394
|
+
resolveResult({ id: node.id, ok: true, value: null });
|
|
395
|
+
else if (record.state === "failed" || record.state === "cancelled")
|
|
396
|
+
resolveResult({ id: node.id, ok: false, error: { code: record.state === "cancelled" ? "CANCELLED" : "AGENT_FAILED", message: `Persisted agent ${record.state}` } });
|
|
397
|
+
}
|
|
398
|
+
for (const node of this.#nodes.values())
|
|
399
|
+
if (node.runId === runId && node.parentId)
|
|
400
|
+
this.#nodes.get(node.parentId)?.children.add(node.id);
|
|
401
|
+
}
|
|
402
|
+
async flush() { await this.#persistence; }
|
|
403
|
+
#inherit(parent, options) {
|
|
404
|
+
const unknown = Object.keys(options).find((key) => !["label", "requestedLabel", "parentBreadcrumb", "cwd", "tools", "worktreeOwner", "model", "thinking", "role", "schema", "retries", "timeoutMs"].includes(key));
|
|
405
|
+
if (unknown)
|
|
406
|
+
throw new WorkflowError("INVALID_METADATA", `Unsupported child agent option: ${unknown}`);
|
|
407
|
+
if (!options.label.trim() || !options.cwd || !Array.isArray(options.tools))
|
|
408
|
+
throw new WorkflowError("INVALID_METADATA", "Agents require label, cwd, and tools");
|
|
409
|
+
if (!parent)
|
|
410
|
+
return Object.freeze({ ...options, tools: Object.freeze([...options.tools]) });
|
|
411
|
+
if (options.cwd !== parent.options.cwd)
|
|
412
|
+
throw new WorkflowError("UNKNOWN_TOOL", "Child cwd cannot differ from its parent");
|
|
413
|
+
const forbidden = options.tools.find((tool) => !parent.options.tools.includes(tool));
|
|
414
|
+
if (forbidden)
|
|
415
|
+
throw new WorkflowError("UNKNOWN_TOOL", `Child tool escalates parent boundary: ${forbidden}`);
|
|
416
|
+
return Object.freeze({ ...options, cwd: parent.options.cwd, tools: Object.freeze([...options.tools]), ...(parent.options.worktreeOwner ? { worktreeOwner: parent.options.worktreeOwner } : {}) });
|
|
417
|
+
}
|
|
418
|
+
/* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-argument, @typescript-eslint/restrict-template-expressions */
|
|
419
|
+
#enqueue(runId, start) { this.#runs.get(runId)?.queue.push(start); this.#dispatch(); }
|
|
420
|
+
#dispatch() {
|
|
421
|
+
while (this.#active < this.sessionLimit && this.#runOrder.length) {
|
|
422
|
+
let selected;
|
|
423
|
+
for (let checked = 0; checked < this.#runOrder.length; checked += 1) {
|
|
424
|
+
const index = (this.#cursor + checked) % this.#runOrder.length;
|
|
425
|
+
const id = this.#runOrder[index];
|
|
426
|
+
const run = id ? this.#runs.get(id) : undefined;
|
|
427
|
+
if (id && run && run.active < run.limit && run.queue.length) {
|
|
428
|
+
selected = id;
|
|
429
|
+
this.#cursor = (index + 1) % this.#runOrder.length;
|
|
430
|
+
break;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
if (!selected)
|
|
434
|
+
return;
|
|
435
|
+
const run = this.#runs.get(selected);
|
|
436
|
+
const start = run.queue.shift();
|
|
437
|
+
run.active += 1;
|
|
438
|
+
this.#active += 1;
|
|
439
|
+
start();
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
#release(runId) {
|
|
443
|
+
const run = this.#runs.get(runId);
|
|
444
|
+
if (run && run.active > 0) {
|
|
445
|
+
run.active -= 1;
|
|
446
|
+
this.#active -= 1;
|
|
447
|
+
this.#dispatch();
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
#settle(node, result) {
|
|
451
|
+
if (["completed", "failed", "cancelled"].includes(node.state))
|
|
452
|
+
return;
|
|
453
|
+
const heldPermit = node.state === "running";
|
|
454
|
+
node.state = result.ok ? "completed" : result.error.code === "CANCELLED" ? "cancelled" : "failed";
|
|
455
|
+
this.#persist(node.runId);
|
|
456
|
+
if (heldPermit)
|
|
457
|
+
this.#release(node.runId);
|
|
458
|
+
for (const childId of node.children) {
|
|
459
|
+
const child = this.#nodes.get(childId);
|
|
460
|
+
if (child && !child.collected)
|
|
461
|
+
this.#cancelTree(child);
|
|
462
|
+
}
|
|
463
|
+
node.resolve(result);
|
|
464
|
+
}
|
|
465
|
+
#cancelTree(node) {
|
|
466
|
+
if (["completed", "failed", "cancelled"].includes(node.state))
|
|
467
|
+
return;
|
|
468
|
+
node.controller.abort();
|
|
469
|
+
for (const childId of node.children) {
|
|
470
|
+
const child = this.#nodes.get(childId);
|
|
471
|
+
if (child)
|
|
472
|
+
this.#cancelTree(child);
|
|
473
|
+
}
|
|
474
|
+
if (node.state === "queued" || node.restored)
|
|
475
|
+
this.#settle(node, { id: node.id, ok: false, error: { code: "CANCELLED", message: "Agent cancelled" } });
|
|
476
|
+
}
|
|
477
|
+
#node(id) {
|
|
478
|
+
const node = this.#nodes.get(id);
|
|
479
|
+
if (!node)
|
|
480
|
+
throw new WorkflowError("UNKNOWN_AGENT_TYPE", `Unknown owned agent: ${id}`);
|
|
481
|
+
return node;
|
|
482
|
+
}
|
|
483
|
+
#persist(runId) {
|
|
484
|
+
if (!this.writeOwnership)
|
|
485
|
+
return;
|
|
486
|
+
const ownership = this.snapshot().filter(({ id }) => id.startsWith(`${runId}:`));
|
|
487
|
+
this.#persistence = this.#persistence.then(() => this.writeOwnership?.(runId, ownership)).then(() => undefined);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
function resolvePath(path) { return path.replace(/[\\/]+$/, ""); }
|
|
491
|
+
function requiredFile(file) {
|
|
492
|
+
if (!file)
|
|
493
|
+
throw new WorkflowError("INTERNAL_ERROR", "Workflow agents require persisted native Pi sessions");
|
|
494
|
+
return file;
|
|
495
|
+
}
|
|
496
|
+
function remaining(timeoutMs, started) {
|
|
497
|
+
return timeoutMs === null || timeoutMs === undefined ? timeoutMs : Math.max(1, timeoutMs - (Date.now() - started));
|
|
498
|
+
}
|
|
499
|
+
function providerLimited(error) {
|
|
500
|
+
if (!error || typeof error !== "object")
|
|
501
|
+
return false;
|
|
502
|
+
const candidate = error;
|
|
503
|
+
return candidate.status === 429 || candidate.code === 429 || candidate.code === "rate_limit_exceeded" || candidate.code === "RATE_LIMITED";
|
|
504
|
+
}
|
|
505
|
+
async function promptWithProviderPause(session, text, timeoutMs, signal, pause) {
|
|
506
|
+
for (;;) {
|
|
507
|
+
try {
|
|
508
|
+
await withTimeout(session.prompt(text), timeoutMs, signal, session);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
catch (error) {
|
|
512
|
+
if (!pause || !providerLimited(error))
|
|
513
|
+
throw error;
|
|
514
|
+
await pause();
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
async function withTimeout(work, timeoutMs, signal, session) {
|
|
519
|
+
if (signal?.aborted)
|
|
520
|
+
throw new WorkflowError("CANCELLED", "Agent cancelled");
|
|
521
|
+
let timer;
|
|
522
|
+
let abort;
|
|
523
|
+
const state = { interrupted: false };
|
|
524
|
+
const timeout = timeoutMs ? new Promise((_, reject) => { timer = setTimeout(() => { state.interrupted = true; reject(new WorkflowError("AGENT_TIMEOUT", "Agent attempt timed out")); }, timeoutMs); }) : new Promise(() => { });
|
|
525
|
+
const cancelled = signal ? new Promise((_, reject) => { abort = () => { state.interrupted = true; reject(new WorkflowError("CANCELLED", "Agent cancelled")); }; signal.addEventListener("abort", abort, { once: true }); }) : new Promise(() => { });
|
|
526
|
+
try {
|
|
527
|
+
await Promise.race([work, timeout, cancelled]);
|
|
528
|
+
}
|
|
529
|
+
finally {
|
|
530
|
+
if (timer)
|
|
531
|
+
clearTimeout(timer);
|
|
532
|
+
if (abort)
|
|
533
|
+
signal?.removeEventListener("abort", abort);
|
|
534
|
+
if (state.interrupted)
|
|
535
|
+
await session.abort?.();
|
|
536
|
+
}
|
|
537
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
export declare const AMBIENT_OPT_IN = "PI_WORKFLOW_EVAL_AMBIENT";
|
|
2
|
+
export declare const AMBIENT_CAPTURE_NOTE = "Ambient Tier D uses the explicit capture extension. Pi 0.80.6 orders CLI extensions before discovered extensions and the first tool registration wins, so ambient tools and skills remain available while workflow execution stays capture-only.";
|
|
3
|
+
export declare const AMBIENT_INVOCATION_MODE = "ambient-capture-only";
|
|
4
|
+
export interface AmbientEvalCase {
|
|
5
|
+
id: string;
|
|
6
|
+
prompt: string;
|
|
7
|
+
timeoutMs: number;
|
|
8
|
+
maxCost: number;
|
|
9
|
+
}
|
|
10
|
+
export declare const AMBIENT_WORKFLOW_EVAL_CASES: readonly AmbientEvalCase[];
|
|
11
|
+
export interface AmbientFixtureRepository {
|
|
12
|
+
root: string;
|
|
13
|
+
fixtureRoot: string;
|
|
14
|
+
worktreesRoot: string;
|
|
15
|
+
fixtureFiles: readonly string[];
|
|
16
|
+
}
|
|
17
|
+
export interface AmbientCaseWorktree {
|
|
18
|
+
id: string;
|
|
19
|
+
path: string;
|
|
20
|
+
fixtureFiles: readonly string[];
|
|
21
|
+
gitStatusBefore: readonly string[];
|
|
22
|
+
}
|
|
23
|
+
export interface AmbientCleanup {
|
|
24
|
+
processExited: boolean;
|
|
25
|
+
processGroupTerminated: boolean;
|
|
26
|
+
worktreeRemoved: boolean;
|
|
27
|
+
fixtureRepoRemoved: boolean;
|
|
28
|
+
tempRootRemoved: boolean;
|
|
29
|
+
captureIdentityVerified: boolean;
|
|
30
|
+
realWorkflowAgentsLaunched: number;
|
|
31
|
+
}
|
|
32
|
+
export interface AmbientManifest {
|
|
33
|
+
invocationMode: string;
|
|
34
|
+
fixtureRoot: string;
|
|
35
|
+
worktreePath: string;
|
|
36
|
+
ambientAgentDir: string;
|
|
37
|
+
fixtureFileList: readonly string[];
|
|
38
|
+
gitStatusBefore: readonly string[];
|
|
39
|
+
gitStatusAfter: readonly string[];
|
|
40
|
+
parentToolSequence: readonly string[];
|
|
41
|
+
skillReads: readonly string[];
|
|
42
|
+
workflowCalls: readonly unknown[];
|
|
43
|
+
workflowCallCount: number;
|
|
44
|
+
tokenCost: {
|
|
45
|
+
input: number;
|
|
46
|
+
output: number;
|
|
47
|
+
cacheRead: number;
|
|
48
|
+
cacheWrite: number;
|
|
49
|
+
totalTokens: number;
|
|
50
|
+
cost: number;
|
|
51
|
+
models: readonly {
|
|
52
|
+
model: string;
|
|
53
|
+
cost: number;
|
|
54
|
+
}[];
|
|
55
|
+
};
|
|
56
|
+
cleanup: AmbientCleanup;
|
|
57
|
+
}
|
|
58
|
+
export interface AmbientCaseResult {
|
|
59
|
+
id: string;
|
|
60
|
+
status: "passed" | "failed" | "timed_out" | "budget_exceeded";
|
|
61
|
+
workflows: readonly unknown[];
|
|
62
|
+
accounting: AmbientManifest["tokenCost"];
|
|
63
|
+
accountingTrustworthy: boolean;
|
|
64
|
+
diagnostics: readonly string[];
|
|
65
|
+
errors: readonly string[];
|
|
66
|
+
manifest: AmbientManifest;
|
|
67
|
+
}
|
|
68
|
+
export interface AmbientWorkflowEvalRunOptions {
|
|
69
|
+
cases?: readonly AmbientEvalCase[];
|
|
70
|
+
caseIds?: readonly string[];
|
|
71
|
+
provider?: string;
|
|
72
|
+
model?: string;
|
|
73
|
+
thinking?: string;
|
|
74
|
+
piCommand?: string;
|
|
75
|
+
artifactsDir?: string;
|
|
76
|
+
environment?: NodeJS.ProcessEnv;
|
|
77
|
+
}
|
|
78
|
+
export interface AmbientWorkflowEvalRunResult {
|
|
79
|
+
artifactDir: string;
|
|
80
|
+
cases: readonly AmbientCaseResult[];
|
|
81
|
+
spent: number;
|
|
82
|
+
mode: string;
|
|
83
|
+
}
|
|
84
|
+
export declare function createAmbientFixtureRepository(parent?: string): AmbientFixtureRepository;
|
|
85
|
+
export declare function createAmbientCaseWorktree(repository: AmbientFixtureRepository, id: string): AmbientCaseWorktree;
|
|
86
|
+
export declare function removeAmbientCaseWorktree(repository: AmbientFixtureRepository, worktree: AmbientCaseWorktree): boolean;
|
|
87
|
+
export declare function removeAmbientFixtureRepository(repository: AmbientFixtureRepository): boolean;
|
|
88
|
+
export interface AmbientPiProcessInput {
|
|
89
|
+
worktree: string;
|
|
90
|
+
sessionDir: string;
|
|
91
|
+
sessionId?: string;
|
|
92
|
+
prompt: string;
|
|
93
|
+
provider: string;
|
|
94
|
+
model: string;
|
|
95
|
+
thinking?: string;
|
|
96
|
+
piCommand?: string;
|
|
97
|
+
timeoutMs: number;
|
|
98
|
+
maxCost: number;
|
|
99
|
+
environment?: NodeJS.ProcessEnv;
|
|
100
|
+
}
|
|
101
|
+
export interface AmbientPiProcessResult {
|
|
102
|
+
exitCode: number | null;
|
|
103
|
+
timedOut: boolean;
|
|
104
|
+
budgetExceeded: boolean;
|
|
105
|
+
processGroupTerminated: boolean;
|
|
106
|
+
stdout: string;
|
|
107
|
+
stderr: string;
|
|
108
|
+
}
|
|
109
|
+
export declare function runAmbientPiProcess(input: AmbientPiProcessInput): Promise<AmbientPiProcessResult>;
|
|
110
|
+
export declare function assertAmbientOptIn(environment?: NodeJS.ProcessEnv): void;
|
|
111
|
+
export declare function runAmbientWorkflowEvals(options?: AmbientWorkflowEvalRunOptions): Promise<AmbientWorkflowEvalRunResult>;
|
|
112
|
+
export declare function formatAmbientSummary(result: AmbientWorkflowEvalRunResult): string;
|