pi-extensible-workflows 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/src/agent-execution.d.ts +13 -0
- package/dist/src/agent-execution.js +177 -40
- package/dist/src/ambient-workflow-evals.js +14 -33
- package/dist/src/cli.d.ts +9 -0
- package/dist/src/cli.js +530 -6
- package/dist/src/doctor.d.ts +4 -4
- package/dist/src/doctor.js +9 -31
- package/dist/src/herdr.d.ts +12 -0
- package/dist/src/herdr.js +74 -0
- package/dist/src/index.d.ts +107 -27
- package/dist/src/index.js +838 -322
- package/dist/src/persistence.d.ts +28 -0
- package/dist/src/persistence.js +213 -16
- package/dist/src/session-inspector.d.ts +1 -0
- package/dist/src/session-inspector.js +3 -0
- package/dist/src/workflow-evals.d.ts +1 -0
- package/dist/src/workflow-evals.js +25 -26
- package/package.json +8 -4
- package/skills/pi-extensible-workflows/SKILL.md +48 -60
- package/src/agent-execution.ts +140 -27
- package/src/ambient-workflow-evals.ts +18 -28
- package/src/cli.ts +418 -6
- package/src/doctor.ts +13 -35
- package/src/herdr.ts +73 -0
- package/src/index.ts +734 -266
- package/src/persistence.ts +192 -16
- package/src/session-inspector.ts +4 -0
- package/src/workflow-evals.ts +15 -17
package/src/persistence.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { execFile } from "node:child_process";
|
|
3
|
+
import { renameSync, rmSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { access, link, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
4
5
|
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
5
6
|
import { homedir } from "node:os";
|
|
@@ -25,6 +26,7 @@ export type PendingWorkflowDecision = BudgetApprovalRequest
|
|
|
25
26
|
export type PersistedOwnershipNode = OwnershipRecord
|
|
26
27
|
type Journal = { completed: Record<string, CompletedOperation>; awaiting?: Record<string, AwaitingCheckpoint>; decisions?: Record<string, PendingWorkflowDecision> };
|
|
27
28
|
export interface WorktreeReference { owner: string; path: string; branch: string; cwd: string; base: string }
|
|
29
|
+
export interface BorrowedWorktreeBinding { name: string; sourceRunId: string; owner: string }
|
|
28
30
|
|
|
29
31
|
const execute = promisify(execFile);
|
|
30
32
|
const gitIdentity = {
|
|
@@ -158,10 +160,28 @@ export function structuralPath(...names: string[]): string {
|
|
|
158
160
|
return names.map((name) => encodeURIComponent(name)).join("/");
|
|
159
161
|
}
|
|
160
162
|
|
|
161
|
-
|
|
163
|
+
export function atomicWriteFile(path: string, content: string): Promise<void>;
|
|
164
|
+
export function atomicWriteFile(path: string, content: string, sync: true): void;
|
|
165
|
+
export function atomicWriteFile(path: string, content: string, sync = false): Promise<void> | void {
|
|
162
166
|
const temporary = `${path}.${String(process.pid)}.${randomUUID()}.tmp`;
|
|
163
|
-
|
|
164
|
-
|
|
167
|
+
if (sync) {
|
|
168
|
+
try {
|
|
169
|
+
writeFileSync(temporary, content, { encoding: "utf8", mode: 0o600 });
|
|
170
|
+
renameSync(temporary, path);
|
|
171
|
+
} catch (error) {
|
|
172
|
+
try { rmSync(temporary, { force: true }); } catch { /* Preserve the original write error. */ }
|
|
173
|
+
throw error;
|
|
174
|
+
}
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
return writeFile(temporary, content, { encoding: "utf8", mode: 0o600 }).then(() => rename(temporary, path)).catch(async (error: unknown) => {
|
|
178
|
+
try { await rm(temporary, { force: true }); } catch { /* Preserve the original write error. */ }
|
|
179
|
+
throw error;
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async function atomicJson(path: string, value: unknown): Promise<void> {
|
|
184
|
+
await atomicWriteFile(path, `${JSON.stringify(value)}\n`);
|
|
165
185
|
}
|
|
166
186
|
|
|
167
187
|
async function json<T>(path: string): Promise<T> { return JSON.parse(await readFile(path, "utf8")) as T; }
|
|
@@ -172,12 +192,13 @@ export class RunStore {
|
|
|
172
192
|
// ponytail: serializes one RunStore instance; cross-process run sharing remains unsupported.
|
|
173
193
|
private stateWrite: Promise<void> = Promise.resolve();
|
|
174
194
|
private worktreeWrite: Promise<void> = Promise.resolve();
|
|
195
|
+
private borrowedWorktreeWrite: Promise<void> = Promise.resolve();
|
|
175
196
|
private snapshotWrite: Promise<void> = Promise.resolve();
|
|
176
197
|
private launchSnapshotWrite: Promise<void> = Promise.resolve();
|
|
177
198
|
// ponytail: the session lease prevents concurrent RunStore writers for one run.
|
|
178
199
|
private systemPromptWrite: Promise<void> = Promise.resolve();
|
|
179
200
|
private conversationWrite: Promise<void> = Promise.resolve();
|
|
180
|
-
constructor(readonly cwd: string, readonly sessionId: string, readonly runId: string, home = homedir()) {
|
|
201
|
+
constructor(readonly cwd: string, readonly sessionId: string, readonly runId: string, readonly home = homedir()) {
|
|
181
202
|
this.cwd = resolve(cwd);
|
|
182
203
|
this.directory = join(runsDirectory(this.cwd, sessionId, home), safePart(runId));
|
|
183
204
|
}
|
|
@@ -193,6 +214,7 @@ export class RunStore {
|
|
|
193
214
|
await atomicJson(join(temporary, "journal.json"), { completed: {}, awaiting: {}, decisions: {} });
|
|
194
215
|
await atomicJson(join(temporary, "ownership.json"), []);
|
|
195
216
|
await atomicJson(join(temporary, "worktrees.json"), []);
|
|
217
|
+
await atomicJson(join(temporary, "borrowed-worktrees.json"), []);
|
|
196
218
|
await atomicJson(join(temporary, "state.json"), run);
|
|
197
219
|
await atomicJson(join(temporary, "system-prompts.json"), { version: 1, entries: [] });
|
|
198
220
|
await atomicJson(join(temporary, "conversations.json"), { version: 1, conversations: {} });
|
|
@@ -245,7 +267,7 @@ export class RunStore {
|
|
|
245
267
|
}
|
|
246
268
|
|
|
247
269
|
async appendEvent(event: WorkflowRunEvent): Promise<void> {
|
|
248
|
-
await this.updateState((run) => ({ ...run, events: [...(run.events ?? []), ...(run.events?.some((current) => current.message === event.message) ? [] : [event])] }));
|
|
270
|
+
await this.updateState((run) => ({ ...run, events: [...(run.events ?? []), ...(run.events?.some((current) => current.type === event.type && current.message === event.message) ? [] : [event])] }));
|
|
249
271
|
}
|
|
250
272
|
|
|
251
273
|
async saveOwnership(nodes: readonly PersistedOwnershipNode[]): Promise<void> {
|
|
@@ -373,6 +395,24 @@ export class RunStore {
|
|
|
373
395
|
return join(this.directory, `worktree-${key}.creating`);
|
|
374
396
|
}
|
|
375
397
|
|
|
398
|
+
private namedWorktreeOwner(name: string): string {
|
|
399
|
+
if (!name.trim()) throw new WorkflowError("WORKTREE_FAILED", "Named worktree names must be non-empty");
|
|
400
|
+
return structuralPath("worktree", "named", name.trim());
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
private worktreeName(owner: string): string | undefined {
|
|
404
|
+
const prefix = `${structuralPath("worktree", "named")}/`;
|
|
405
|
+
if (!owner.startsWith(prefix)) return undefined;
|
|
406
|
+
const encoded = owner.slice(prefix.length);
|
|
407
|
+
if (!encoded || encoded.includes("/")) return undefined;
|
|
408
|
+
try {
|
|
409
|
+
const name = decodeURIComponent(encoded);
|
|
410
|
+
return name.trim() ? name : undefined;
|
|
411
|
+
} catch {
|
|
412
|
+
return undefined;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
376
416
|
private structuralWorktree(owner: string, record: unknown): WorktreeReference {
|
|
377
417
|
if (!record || typeof record !== "object") throw new Error(`Invalid worktree record for ${owner}`);
|
|
378
418
|
const candidate = record as Partial<WorktreeReference>;
|
|
@@ -383,6 +423,110 @@ export class RunStore {
|
|
|
383
423
|
return candidate as WorktreeReference;
|
|
384
424
|
}
|
|
385
425
|
|
|
426
|
+
private async borrowedWorktreeRecords(wait = true): Promise<readonly BorrowedWorktreeBinding[]> {
|
|
427
|
+
if (wait) await this.borrowedWorktreeWrite;
|
|
428
|
+
const records = await json<unknown[]>(join(this.directory, "borrowed-worktrees.json")).catch((error: unknown) => { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; });
|
|
429
|
+
if (!Array.isArray(records)) throw new WorkflowError("WORKTREE_FAILED", "Borrowed worktree bindings are invalid");
|
|
430
|
+
const seen = new Set<string>();
|
|
431
|
+
return records.map((record) => {
|
|
432
|
+
if (!record || typeof record !== "object") throw new WorkflowError("WORKTREE_FAILED", "Borrowed worktree binding is invalid");
|
|
433
|
+
const candidate = record as Partial<BorrowedWorktreeBinding>;
|
|
434
|
+
if (typeof candidate.name !== "string" || !candidate.name.trim() || candidate.name !== candidate.name.trim() || typeof candidate.sourceRunId !== "string" || !candidate.sourceRunId || typeof candidate.owner !== "string" || candidate.owner !== this.namedWorktreeOwner(candidate.name)) throw new WorkflowError("WORKTREE_FAILED", "Borrowed worktree binding is invalid");
|
|
435
|
+
if (seen.has(candidate.name)) throw new WorkflowError("WORKTREE_FAILED", `Duplicate borrowed worktree binding for ${candidate.name}`);
|
|
436
|
+
seen.add(candidate.name);
|
|
437
|
+
return { name: candidate.name, sourceRunId: candidate.sourceRunId, owner: candidate.owner };
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
async borrowedWorktrees(): Promise<readonly BorrowedWorktreeBinding[]> { return this.borrowedWorktreeRecords(); }
|
|
442
|
+
|
|
443
|
+
private async borrowedWorktree(name: string): Promise<BorrowedWorktreeBinding | undefined> {
|
|
444
|
+
return (await this.borrowedWorktreeRecords()).find((binding) => binding.name === name);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
private async sourceRun(sourceRunId: string): Promise<RunStore> {
|
|
448
|
+
if (!sourceRunId || sourceRunId === this.runId) throw new WorkflowError("WORKTREE_FAILED", "Borrowed worktree source run is invalid");
|
|
449
|
+
const source = new RunStore(this.cwd, this.sessionId, sourceRunId, this.home);
|
|
450
|
+
try {
|
|
451
|
+
const loaded = await source.load();
|
|
452
|
+
if (!["completed", "failed", "stopped"].includes(loaded.run.state)) throw new Error(`Source run ${sourceRunId} is not terminal`);
|
|
453
|
+
return source;
|
|
454
|
+
} catch (error) {
|
|
455
|
+
if (error instanceof WorkflowError && error.code === "WORKTREE_FAILED") throw error;
|
|
456
|
+
throw new WorkflowError("WORKTREE_FAILED", error instanceof Error ? error.message : String(error));
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
async validateParentRun(parentRunId: string): Promise<void> { await this.sourceRun(parentRunId); }
|
|
461
|
+
|
|
462
|
+
private async ownedWorktree(owner: string, cwd?: string): Promise<WorktreeReference> {
|
|
463
|
+
const records = await json<unknown[]>(join(this.directory, "worktrees.json"));
|
|
464
|
+
const matches = records.filter((candidate) => candidate && typeof candidate === "object" && (candidate as Partial<WorktreeReference>).owner === owner);
|
|
465
|
+
if (matches.length !== 1) throw new Error(`Missing or duplicate worktree record for ${owner}`);
|
|
466
|
+
const record = this.structuralWorktree(owner, matches[0]);
|
|
467
|
+
if (cwd !== undefined && resolve(cwd) !== resolve(record.cwd)) throw new Error(`Invalid worktree record for ${owner}`);
|
|
468
|
+
await access(record.cwd);
|
|
469
|
+
return record;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
private async resolveBorrowedWorktree(binding: BorrowedWorktreeBinding, seen: Set<string>): Promise<{ reference: WorktreeReference; sourceRunId: string; owner: string }> {
|
|
473
|
+
try {
|
|
474
|
+
const source = await this.sourceRun(binding.sourceRunId);
|
|
475
|
+
const resolved = await source.findNamedWorktree(binding.name, seen);
|
|
476
|
+
if (!resolved) throw new Error(`Missing named worktree ${binding.name} in source run ${binding.sourceRunId}`);
|
|
477
|
+
if (resolved.owner !== binding.owner) throw new Error(`Borrowed worktree binding does not match source owner for ${binding.name}`);
|
|
478
|
+
return resolved;
|
|
479
|
+
} catch (error) {
|
|
480
|
+
throw error instanceof WorkflowError && error.code === "WORKTREE_FAILED" ? error : new WorkflowError("WORKTREE_FAILED", error instanceof Error ? error.message : String(error));
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
private async findNamedWorktree(name: string, seen: Set<string> = new Set()): Promise<{ reference: WorktreeReference; sourceRunId: string; owner: string } | undefined> {
|
|
485
|
+
const owner = this.namedWorktreeOwner(name);
|
|
486
|
+
if (seen.has(this.runId)) throw new WorkflowError("WORKTREE_FAILED", "Borrowed worktree bindings contain a cycle");
|
|
487
|
+
const nextSeen = new Set(seen);
|
|
488
|
+
nextSeen.add(this.runId);
|
|
489
|
+
const binding = await this.borrowedWorktree(name);
|
|
490
|
+
if (binding) {
|
|
491
|
+
const loaded = await this.load();
|
|
492
|
+
if (loaded.run.parentRunId === undefined) throw new WorkflowError("WORKTREE_FAILED", `Borrowed worktree ${name} has no parent run`);
|
|
493
|
+
const parent = await this.sourceRun(loaded.run.parentRunId);
|
|
494
|
+
const resolved = await parent.findNamedWorktree(name, nextSeen);
|
|
495
|
+
if (!resolved || resolved.sourceRunId !== binding.sourceRunId || resolved.owner !== binding.owner) throw new WorkflowError("WORKTREE_FAILED", `Borrowed worktree binding for ${name} is not inherited from its parent run`);
|
|
496
|
+
return resolved;
|
|
497
|
+
}
|
|
498
|
+
const records = await json<unknown[]>(join(this.directory, "worktrees.json"));
|
|
499
|
+
const matches = records.filter((candidate) => candidate && typeof candidate === "object" && (candidate as Partial<WorktreeReference>).owner === owner);
|
|
500
|
+
if (matches.length === 0) return undefined;
|
|
501
|
+
try {
|
|
502
|
+
const reference = await this.ownedWorktree(owner);
|
|
503
|
+
return { reference, sourceRunId: this.runId, owner };
|
|
504
|
+
} catch (error) {
|
|
505
|
+
throw new WorkflowError("WORKTREE_FAILED", error instanceof Error ? error.message : String(error));
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
async resolveNamedWorktree(name: string, seen: Set<string> = new Set()): Promise<{ reference: WorktreeReference; sourceRunId: string; owner: string }> {
|
|
510
|
+
const resolved = await this.findNamedWorktree(name, seen);
|
|
511
|
+
if (!resolved) throw new WorkflowError("WORKTREE_FAILED", `Missing named worktree ${name}`);
|
|
512
|
+
return resolved;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
async validateBorrowedWorktrees(): Promise<void> {
|
|
516
|
+
try {
|
|
517
|
+
const loaded = await this.load();
|
|
518
|
+
if (loaded.run.parentRunId !== undefined) await this.validateParentRun(loaded.run.parentRunId);
|
|
519
|
+
for (const binding of await this.borrowedWorktreeRecords()) await this.resolveBorrowedWorktree(binding, new Set([this.runId]));
|
|
520
|
+
} catch (error) {
|
|
521
|
+
throw error instanceof WorkflowError && error.code === "WORKTREE_FAILED" ? error : new WorkflowError("WORKTREE_FAILED", error instanceof Error ? error.message : String(error));
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
async ownsWorktree(owner: string): Promise<boolean> {
|
|
526
|
+
const records = await json<unknown[]>(join(this.directory, "worktrees.json"));
|
|
527
|
+
return records.filter((candidate) => candidate && typeof candidate === "object" && (candidate as Partial<WorktreeReference>).owner === owner).length === 1;
|
|
528
|
+
}
|
|
529
|
+
|
|
386
530
|
private async cleanupMarker(markerPath: string): Promise<void> {
|
|
387
531
|
let marker: Partial<{ owner: string; path: string; branch: string; base: string }>;
|
|
388
532
|
try { marker = await json(markerPath); } catch { return; }
|
|
@@ -407,13 +551,14 @@ export class RunStore {
|
|
|
407
551
|
async validateWorktree(owner: string, cwd?: string): Promise<WorktreeReference> {
|
|
408
552
|
try {
|
|
409
553
|
await this.load();
|
|
410
|
-
const
|
|
411
|
-
const
|
|
412
|
-
if (
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
554
|
+
const name = this.worktreeName(owner);
|
|
555
|
+
const binding = name ? await this.borrowedWorktree(name) : undefined;
|
|
556
|
+
if (binding) {
|
|
557
|
+
const resolved = await this.resolveBorrowedWorktree(binding, new Set([this.runId]));
|
|
558
|
+
if (cwd !== undefined && resolve(cwd) !== resolve(resolved.reference.cwd)) throw new Error(`Invalid worktree record for ${owner}`);
|
|
559
|
+
return resolved.reference;
|
|
560
|
+
}
|
|
561
|
+
return await this.ownedWorktree(owner, cwd);
|
|
417
562
|
} catch (error) {
|
|
418
563
|
throw error instanceof WorkflowError && error.code === "WORKTREE_FAILED" ? error : new WorkflowError("WORKTREE_FAILED", error instanceof Error ? error.message : String(error));
|
|
419
564
|
}
|
|
@@ -421,9 +566,19 @@ export class RunStore {
|
|
|
421
566
|
|
|
422
567
|
async worktree(owner: string): Promise<WorktreeReference> {
|
|
423
568
|
const write = this.worktreeWrite.then(async () => {
|
|
424
|
-
await this.load();
|
|
569
|
+
const loaded = await this.load();
|
|
425
570
|
const recordsPath = join(this.directory, "worktrees.json");
|
|
426
571
|
let records = await json<WorktreeReference[]>(recordsPath).catch((error: unknown) => { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; });
|
|
572
|
+
const name = this.worktreeName(owner);
|
|
573
|
+
const binding = name ? await this.borrowedWorktree(name) : undefined;
|
|
574
|
+
if (binding) return (await this.resolveBorrowedWorktree(binding, new Set([this.runId]))).reference;
|
|
575
|
+
if (name && loaded.run.parentRunId !== undefined) {
|
|
576
|
+
const resolved = await this.resolveNamedWorktreeFromParent(name, loaded.run.parentRunId);
|
|
577
|
+
if (resolved) {
|
|
578
|
+
await this.bindBorrowedWorktree({ name, sourceRunId: resolved.sourceRunId, owner: resolved.owner });
|
|
579
|
+
return resolved.reference;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
427
582
|
const existing = records.find((record) => record.owner === owner);
|
|
428
583
|
if (existing) return this.validateWorktree(owner);
|
|
429
584
|
const { path, branch } = this.expectedWorktree(owner);
|
|
@@ -469,6 +624,25 @@ export class RunStore {
|
|
|
469
624
|
return write;
|
|
470
625
|
}
|
|
471
626
|
|
|
627
|
+
private async resolveNamedWorktreeFromParent(name: string, parentRunId: string): Promise<{ reference: WorktreeReference; sourceRunId: string; owner: string } | undefined> {
|
|
628
|
+
const source = await this.sourceRun(parentRunId);
|
|
629
|
+
return source.findNamedWorktree(name, new Set([this.runId]));
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
private async bindBorrowedWorktree(binding: BorrowedWorktreeBinding): Promise<void> {
|
|
633
|
+
const write = this.borrowedWorktreeWrite.then(async () => {
|
|
634
|
+
const records = [...await this.borrowedWorktreeRecords(false)];
|
|
635
|
+
const existing = records.find((candidate) => candidate.name === binding.name);
|
|
636
|
+
if (existing) {
|
|
637
|
+
if (JSON.stringify(existing) !== JSON.stringify(binding)) throw new WorkflowError("WORKTREE_FAILED", `Borrowed worktree binding for ${binding.name} changed`);
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
records.push(binding);
|
|
641
|
+
await atomicJson(join(this.directory, "borrowed-worktrees.json"), records);
|
|
642
|
+
});
|
|
643
|
+
this.borrowedWorktreeWrite = write.then(() => undefined, () => undefined);
|
|
644
|
+
await write;
|
|
645
|
+
}
|
|
472
646
|
async snapshotWorktree(owner: string): Promise<string> {
|
|
473
647
|
try {
|
|
474
648
|
const write = this.snapshotWrite.then(async () => {
|
|
@@ -491,11 +665,13 @@ export class RunStore {
|
|
|
491
665
|
throw error instanceof WorkflowError && error.code === "WORKTREE_FAILED" ? error : new WorkflowError("WORKTREE_FAILED", error instanceof Error ? error.message : String(error));
|
|
492
666
|
}
|
|
493
667
|
}
|
|
494
|
-
|
|
495
668
|
async worktrees(): Promise<readonly WorktreeReference[]> {
|
|
496
669
|
const records = await json<WorktreeReference[]>(join(this.directory, "worktrees.json")).catch((error: unknown) => { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; });
|
|
497
|
-
const
|
|
498
|
-
|
|
670
|
+
const bindings = await this.borrowedWorktreeRecords();
|
|
671
|
+
const boundOwners = new Set(bindings.map((binding) => binding.owner));
|
|
672
|
+
const owned = await Promise.all(records.filter((record) => !boundOwners.has(record.owner)).map(async (record) => { try { return await this.validateWorktree(record.owner); } catch { return undefined; } }));
|
|
673
|
+
const borrowed = await Promise.all(bindings.map(async (binding) => (await this.resolveBorrowedWorktree(binding, new Set([this.runId]))).reference));
|
|
674
|
+
return [...owned.filter((record): record is WorktreeReference => record !== undefined), ...borrowed];
|
|
499
675
|
}
|
|
500
676
|
|
|
501
677
|
async changedWorktrees(): Promise<readonly WorktreeReference[]> {
|
package/src/session-inspector.ts
CHANGED
|
@@ -53,6 +53,10 @@ export function transcriptLines(entries: readonly SessionEntry[]): string[] {
|
|
|
53
53
|
});
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
export function transcriptFileLines(path: string): string[] {
|
|
57
|
+
return transcriptLines(SessionManager.open(path).buildContextEntries());
|
|
58
|
+
}
|
|
59
|
+
|
|
56
60
|
function mergedModels(groups: readonly (readonly ModelUsage[])[]): ModelUsage[] {
|
|
57
61
|
const totals = new Map<string, number>();
|
|
58
62
|
for (const group of groups) for (const item of group) totals.set(item.model, (totals.get(item.model) ?? 0) + item.cost);
|
package/src/workflow-evals.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { Value } from "typebox/value";
|
|
|
8
8
|
import { getAgentDir, parseFrontmatter, SessionManager } from "@earendil-works/pi-coding-agent";
|
|
9
9
|
import { CAPTURE_ERROR_PREFIX, CAPTURE_IDENTITY, resolveWorkflowSkillPath } from "./eval-capture-extension.js";
|
|
10
10
|
export { resolveWorkflowSkillPath } from "./eval-capture-extension.js";
|
|
11
|
-
import { ERROR_CODES, inspectWorkflowScript, loadAgentDefinitions, runWorkflow, WorkflowError, type AgentIdentity, type JsonSchema, type JsonValue, type StaticWorkflowCall, type StaticWorkflowExecution, type WorkflowErrorCode } from "./index.js";
|
|
11
|
+
import { ERROR_CODES, inspectWorkflowScript, isObject, loadAgentDefinitions, runWorkflow, WORKFLOW_CALL_KINDS, WorkflowError, type AgentIdentity, type JsonSchema, type JsonValue, type StaticWorkflowCall, type StaticWorkflowExecution, type WorkflowErrorCode } from "./index.js";
|
|
12
12
|
|
|
13
13
|
export type SignificantAction = { kind: "tool"; name: string } | { kind: "text" } | { kind: "thinking" };
|
|
14
14
|
export type SequenceExpectation = readonly string[] | { equals?: readonly string[]; startsWith?: readonly string[] };
|
|
@@ -95,7 +95,6 @@ export const SAFE_PARENT_EVAL_TOOLS = Object.freeze(["read", "grep", "find", "ba
|
|
|
95
95
|
const EVAL_MODEL_TOKEN = "$EVAL_MODEL";
|
|
96
96
|
const semantic = (description: string): readonly SemanticCriterion[] => [{ id: "intent", description }];
|
|
97
97
|
const JSON_RESULT_TYPES = ["null", "boolean", "number", "integer", "string", "array", "object"] as const;
|
|
98
|
-
const WORKFLOW_CALL_KINDS = ["agent", "conversation", "parallel", "pipeline", "checkpoint", "phase", "withWorktree"] as const;
|
|
99
98
|
const AGENT_OPTION_NAMES = ["role", "model", "thinking", "tools", "retries"] as const;
|
|
100
99
|
const expectationKeys = ["firstSignificantAction", "firstTool", "firstBatchToolSequence", "parentToolSequence", "workflowCallCount", "requiredOperations", "forbiddenOperations", "requiredRoles", "minimumAgentCalls", "requireOutputSchema", "expectedResults", "agentPolicies", "requiredAgentOrder", "requiredAgentStructures", "requiredDataFlow"] as const;
|
|
101
100
|
const caseKeys = ["id", "prompt", "timeoutMs", "maxCost", "expectations", "expectedWorkflowCalls", "semanticCriteria"] as const;
|
|
@@ -130,7 +129,6 @@ export function loadWorkflowEvalCases(directory = evalCasesDirectory()): readonl
|
|
|
130
129
|
export const INITIAL_WORKFLOW_EVAL_CASES: readonly WorkflowEvalCase[] = loadWorkflowEvalCases();
|
|
131
130
|
|
|
132
131
|
|
|
133
|
-
function isObject(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
|
|
134
132
|
function isJson(value: unknown): value is JsonValue { if (value === null || typeof value === "string" || typeof value === "boolean") return true; if (typeof value === "number") return Number.isFinite(value); if (Array.isArray(value)) return value.every(isJson); return isObject(value) && Object.values(value).every(isJson); }
|
|
135
133
|
function asJsonObject(value: unknown): Readonly<Record<string, JsonValue>> | undefined { return isObject(value) && Object.values(value).every(isJson) ? value as Readonly<Record<string, JsonValue>> : undefined; }
|
|
136
134
|
function jsonType(value: JsonValue): JsonResultType { if (value === null) return "null"; if (Array.isArray(value)) return "array"; if (typeof value === "number") return Number.isInteger(value) ? "integer" : "number"; if (typeof value === "object") return "object"; if (typeof value === "string") return "string"; if (typeof value === "boolean") return "boolean"; return "null"; }
|
|
@@ -487,6 +485,7 @@ export async function replayWorkflowScript(script: string, args: JsonValue = nul
|
|
|
487
485
|
return `fake:${prompt}`;
|
|
488
486
|
} finally { active -= 1; }
|
|
489
487
|
},
|
|
488
|
+
worktree: async () => ({ path: "/worktrees/eval", branch: "eval-branch" }),
|
|
490
489
|
phase: (name) => { phases.push(name); },
|
|
491
490
|
log: (message) => { logs.push(message); },
|
|
492
491
|
}, signal);
|
|
@@ -595,7 +594,7 @@ function semanticJudgePrompt(evalCase: WorkflowEvalCase, calls: readonly Capture
|
|
|
595
594
|
const roles = loadAgentDefinitions(cwd, join(home, ".pi", "agent"));
|
|
596
595
|
const usedRoles = new Set(calls.flatMap(({ script }) => { try { return script ? inspectWorkflowScript(script).flatMap((call) => call.kind === "agent" && call.role ? [call.role] : []) : []; } catch { return []; } }));
|
|
597
596
|
const roleText = [...usedRoles].map((role) => `${role}: ${roles[role]?.description ?? "no description"}`).join("\n") || "none";
|
|
598
|
-
const docs = "agent(prompt, options) delegates; parallel(name, tasks) runs independent tasks concurrently; pipeline(name, items, stages) applies ordered stages; prompt(template, data) carries values into prompts. A role owns model/thinking/tools policy.";
|
|
597
|
+
const docs = "agent(prompt, options) delegates; shell(command, options) runs a deterministic host command and returns exitCode/stdout/stderr; parallel(name, tasks) runs independent tasks concurrently; pipeline(name, items, stages) applies ordered stages; prompt(template, data) carries values into prompts. A role owns model/thinking/tools policy.";
|
|
599
598
|
return `Judge whether the captured workflow design satisfies each criterion. Do not execute it. Return only JSON: {"criteria":[{"id":"criterion id","pass":true,"evidence":"specific script evidence"}]}.\n\nOriginal request:\n${evalCase.prompt}\n\nCriteria:\n${JSON.stringify(evalCase.semanticCriteria ?? [])}\n\nDSL:\n${docs}\n\nRelevant roles:\n${roleText}\n\nCaptured workflow script(s):\n${calls.map((call, index) => `--- ${String(index)} ---\n${call.script ?? "<missing>"}`).join("\n")}`;
|
|
600
599
|
}
|
|
601
600
|
|
|
@@ -646,21 +645,20 @@ function seedEvalProject(cwd: string, home: string, model: string): void {
|
|
|
646
645
|
if (frontmatterEnd >= 0) writeFileSync(path, `${content.slice(0, frontmatterEnd).replace(/^model:.*$/m, `model: ${model}`)}${content.slice(frontmatterEnd)}`);
|
|
647
646
|
}
|
|
648
647
|
}
|
|
649
|
-
|
|
648
|
+
export function findSessionFile(directory: string, sessionId: string): string | undefined {
|
|
649
|
+
if (!existsSync(directory)) return undefined;
|
|
650
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
651
|
+
const path = join(directory, entry.name);
|
|
652
|
+
if (entry.isDirectory()) { const found = findSessionFile(path, sessionId); if (found) return found; }
|
|
653
|
+
else if (entry.name.endsWith(".jsonl")) {
|
|
654
|
+
try { const header = JSON.parse(readFileSync(path, "utf8").split("\n")[0] ?? "{}") as unknown; if (isObject(header) && header.id === sessionId) return path; } catch { /* Ignore incomplete sessions. */ }
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
return undefined;
|
|
658
|
+
}
|
|
650
659
|
async function findParentSession(cwd: string, sessionDir: string, sessionId: string): Promise<string | undefined> {
|
|
651
660
|
try { const sessions = await SessionManager.list(cwd, sessionDir); const found = sessions.find((session) => session.id === sessionId); if (found) return found.path; } catch { /* Fall through to the JSONL scan. */ }
|
|
652
|
-
|
|
653
|
-
if (!existsSync(directory)) return undefined;
|
|
654
|
-
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
655
|
-
const path = join(directory, entry.name);
|
|
656
|
-
if (entry.isDirectory()) { const found = visit(path); if (found) return found; }
|
|
657
|
-
else if (entry.name.endsWith(".jsonl")) {
|
|
658
|
-
try { const header = JSON.parse(readFileSync(path, "utf8").split("\n")[0] ?? "{}") as unknown; if (isObject(header) && header.id === sessionId) return path; } catch { /* Ignore incomplete files. */ }
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
return undefined;
|
|
662
|
-
};
|
|
663
|
-
return visit(sessionDir);
|
|
661
|
+
return findSessionFile(sessionDir, sessionId);
|
|
664
662
|
}
|
|
665
663
|
|
|
666
664
|
function emptyAccounting(cost = 0): EvalAccounting { return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost, models: [] }; }
|