pi-crew 0.9.44 → 0.9.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -0
- package/README.md +38 -3
- package/dist/build-meta.json +349 -203
- package/dist/index.mjs +2229 -2968
- package/dist/index.mjs.map +4 -4
- package/docs/decisions/2026-07-21-broker-phase4-default-on.md +77 -0
- package/docs/decisions/2026-07-21-broker-windows-perms.md +91 -0
- package/docs/decisions/2026-07-22-broker-phase4-gated-on.md +99 -0
- package/docs/decisions/README.md +3 -0
- package/docs/publishing.md +26 -0
- package/package.json +3 -1
- package/scripts/build-bundle.mjs +7 -0
- package/scripts/postinstall.mjs +35 -1
- package/scripts/pty_probe.py +174 -0
- package/skills/real-test-pi-crew/SKILL.md +659 -0
- package/src/agents/discover-agents.ts +1 -1
- package/src/config/config.ts +42 -1
- package/src/config/defaults.ts +45 -1
- package/src/config/types.ts +19 -0
- package/src/extension/register.ts +6 -1
- package/src/extension/registration/context-builder.ts +4 -0
- package/src/extension/registration/lifecycle-handlers.ts +200 -6
- package/src/extension/registration/registration-types.ts +9 -0
- package/src/extension/registration/subagent-manager-setup.ts +178 -59
- package/src/extension/run-import.ts +21 -1
- package/src/extension/team-tool/api.ts +4 -2
- package/src/prompt/prompt-runtime.ts +108 -0
- package/src/runtime/async-runner.ts +9 -1
- package/src/runtime/broker-issuer.ts +37 -0
- package/src/runtime/child-pi-spawn.ts +53 -0
- package/src/runtime/child-pi.ts +42 -11
- package/src/runtime/crew-broker-child.ts +88 -0
- package/src/runtime/crew-broker-client.ts +673 -0
- package/src/runtime/crew-broker-tokens.ts +84 -0
- package/src/runtime/crew-broker.ts +1276 -0
- package/src/runtime/dynamic-workflow-context.ts +7 -3
- package/src/runtime/dynamic-workflow-runner.ts +1 -1
- package/src/runtime/manifest-cache.ts +30 -0
- package/src/runtime/plan-templates.ts +8 -6
- package/src/runtime/resilient-edit.ts +16 -15
- package/src/runtime/role-permission.ts +27 -2
- package/src/runtime/run-coalesced-task-group.ts +72 -15
- package/src/runtime/task-packet.ts +1 -1
- package/src/schema/config-schema.ts +14 -0
- package/src/state/event-log.ts +88 -34
- package/src/state/locks.ts +53 -0
- package/src/state/mailbox.ts +208 -4
- package/src/state/run-metrics.ts +40 -12
- package/src/ui/key-utils.ts +42 -0
- package/src/ui/keybinding-map.ts +29 -3
- package/src/ui/live-run-sidebar.ts +1 -9
- package/src/ui/run-dashboard.ts +29 -9
- package/src/ui/settings-overlay.ts +42 -22
- package/src/utils/incremental-reader.ts +105 -0
- package/src/utils/ndjson.ts +115 -0
- package/src/utils/session-utils.ts +30 -0
- package/src/utils/socket-path.ts +127 -0
- package/src/utils/visual.ts +27 -91
- package/workflows/default.workflow.md +1 -1
- package/workflows/fast-fix.workflow.md +1 -1
- package/workflows/plan-execute.workflow.md +1 -1
- package/workflows/review.workflow.md +1 -1
- package/src/runtime/auto-resume.ts +0 -100
- package/src/runtime/notebook-helpers.ts +0 -88
- package/src/runtime/orphan-sentinel.ts +0 -7
|
@@ -236,9 +236,13 @@ export function synthesizeAgentConfig(name: string, model?: string): AgentConfig
|
|
|
236
236
|
description: `Synthesized agent for dynamic workflow (${name}).`,
|
|
237
237
|
source: "dynamic",
|
|
238
238
|
filePath: `<dynamic-workflow>`,
|
|
239
|
-
systemPrompt: `You are ${name}.`,
|
|
239
|
+
systemPrompt: `You are ${name}, an agent in a dynamic pi-crew workflow. Use the provided tools (read, grep, find, ls, bash) to investigate the target and produce concrete written findings. Do not return an empty response — always output substantive content for your task.`,
|
|
240
240
|
model,
|
|
241
|
-
|
|
241
|
+
// Round-N fix: give synthesized agents the STANDARD file-investigation toolkit
|
|
242
|
+
// (matches real built-in agents). Previously tools:[] relied on pi-args default
|
|
243
|
+
// behavior, which left most synthesized agents (explorer/analyst/critic/executor)
|
|
244
|
+
// unable to read the codebase → 10/11 agents returned empty/!ok in distill-dwf runs.
|
|
245
|
+
tools: ["read", "grep", "find", "ls", "bash"],
|
|
242
246
|
inheritProjectContext: false,
|
|
243
247
|
inheritSkills: false,
|
|
244
248
|
};
|
|
@@ -420,7 +424,7 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
420
424
|
// This correctly reduces spent when actualUsage < ESTIMATE.
|
|
421
425
|
wfState.spent += (parsed.usage?.input ?? 0) + (parsed.usage?.output ?? 0) - ESTIMATE;
|
|
422
426
|
reserved = false;
|
|
423
|
-
let text = parsed.finalText
|
|
427
|
+
let text = childResult.rawFinalText || parsed.finalText || "";
|
|
424
428
|
// Round-11 test fix: parsePiJsonOutput only extracts text from pi event stream
|
|
425
429
|
// ({type:"message_end", message:{role:"assistant", content:[...]}}). When the
|
|
426
430
|
// agent emits plain JSON, plain text, or a different format, finalText is empty.
|
|
@@ -208,7 +208,7 @@ export async function runDynamicWorkflow(input: RunDynamicWorkflowInput): Promis
|
|
|
208
208
|
// it does NOT kill the script. Promise.race with a hard timeout at least returns an
|
|
209
209
|
// error so the runner doesn't hang. The spawned child process is leaked, but the
|
|
210
210
|
// dynamic-workflow returns failure promptly. (v1.5: use Worker threads to actually kill.)
|
|
211
|
-
const SCRIPT_TIMEOUT_MS = Number.parseInt(process.env.PI_CREW_DWF_SCRIPT_TIMEOUT_MS ?? "", 10) ||
|
|
211
|
+
const SCRIPT_TIMEOUT_MS = Number.parseInt(process.env.PI_CREW_DWF_SCRIPT_TIMEOUT_MS ?? "", 10) || 1_800_000; // 30 min default (was 10 min; raised for distill-dwf + similar long pipelines)
|
|
212
212
|
let timeoutHandle: NodeJS.Timeout | undefined;
|
|
213
213
|
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
214
214
|
timeoutHandle = setTimeout(() => {
|
|
@@ -164,6 +164,7 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
|
|
|
164
164
|
manifestIndex.clear();
|
|
165
165
|
}
|
|
166
166
|
listCache.clear();
|
|
167
|
+
invalidateListActive();
|
|
167
168
|
}
|
|
168
169
|
|
|
169
170
|
function scheduleListRefresh(): void {
|
|
@@ -174,12 +175,17 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
|
|
|
174
175
|
const timer = listTimer;
|
|
175
176
|
listTimer = undefined;
|
|
176
177
|
listCache.clear();
|
|
178
|
+
invalidateListActive();
|
|
177
179
|
timer?.unref();
|
|
178
180
|
}, ttlMs);
|
|
179
181
|
// Unref immediately so the timer never blocks process exit (defense in
|
|
180
182
|
// depth: the in-callback unref above may not run if shutdown happens
|
|
181
183
|
// before the timer fires).
|
|
182
184
|
listTimer.unref();
|
|
185
|
+
// FIND-03: invalidate the listActive() cache eagerly on every watcher
|
|
186
|
+
// tick. The TTL is the fallback for missed events; the watcher-driven
|
|
187
|
+
// path gives the tightest possible invalidation.
|
|
188
|
+
invalidateListActive();
|
|
183
189
|
}
|
|
184
190
|
|
|
185
191
|
function loadManifest(runId: string, rootsToCheck: string[]): CachedManifest | undefined {
|
|
@@ -266,6 +272,18 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
|
|
|
266
272
|
return undefined;
|
|
267
273
|
}
|
|
268
274
|
|
|
275
|
+
// FIND-03: short-TTL cache for listActive(). Mirrors the listCache pattern
|
|
276
|
+
// used by list(): we cache the un-capped running set and apply the caller's
|
|
277
|
+
// `limit` post-hoc on every return. Storing the full set (not a
|
|
278
|
+
// limit-sliced array) is what preserves the RT-F3 contract — callers with
|
|
279
|
+
// different `limit` values all see the same underlying "every running run"
|
|
280
|
+
// result, never a top-N createdAt-filtered view.
|
|
281
|
+
let listActiveCache: { result: TeamRunManifest[] | null; expiresAt: number } = { result: null, expiresAt: 0 };
|
|
282
|
+
|
|
283
|
+
function invalidateListActive(): void {
|
|
284
|
+
listActiveCache = { result: null, expiresAt: 0 };
|
|
285
|
+
}
|
|
286
|
+
|
|
269
287
|
/**
|
|
270
288
|
* RT-F3: filter to `status === "running"` BEFORE applying the limit so an
|
|
271
289
|
* orphaned run that has been pushed past the top-N by recent successful
|
|
@@ -277,9 +295,19 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
|
|
|
277
295
|
* silently drop "running" runs that fell past the top-N createdAt cutoff.
|
|
278
296
|
* Still goes through parseManifestIfChanged for stat+size memoization, so
|
|
279
297
|
* the per-run I/O cost is the same as list().
|
|
298
|
+
*
|
|
299
|
+
* FIND-03 perf: the full scan is memoized behind a 500ms TTL (same TTL as
|
|
300
|
+
* list()). fs.watch-driven scheduleListRefresh() invalidates the cache
|
|
301
|
+
* immediately so the next call re-scans. The cap is applied AFTER the
|
|
302
|
+
* cache lookup so a cached scan result can be sliced to ANY limit without
|
|
303
|
+
* re-scanning.
|
|
280
304
|
*/
|
|
281
305
|
function listActive(limit: number): TeamRunManifest[] {
|
|
282
306
|
const cap = Math.max(0, limit);
|
|
307
|
+
const now = Date.now();
|
|
308
|
+
if (listActiveCache.result !== null && listActiveCache.expiresAt > now) {
|
|
309
|
+
return listActiveCache.result.slice(0, cap);
|
|
310
|
+
}
|
|
283
311
|
const parsedEntries = [
|
|
284
312
|
...roots.flatMap((root) => collectRoots(root)),
|
|
285
313
|
...activeRunEntries().map((entry) => ({
|
|
@@ -303,6 +331,7 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
|
|
|
303
331
|
.filter((value): value is CachedManifest => value !== undefined)
|
|
304
332
|
.map((value) => value.manifest)
|
|
305
333
|
.filter((manifest) => manifest.status === "running");
|
|
334
|
+
listActiveCache = { result: running, expiresAt: now + ttlMs };
|
|
306
335
|
return running.slice(0, cap);
|
|
307
336
|
}
|
|
308
337
|
|
|
@@ -340,6 +369,7 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
|
|
|
340
369
|
watchers = [];
|
|
341
370
|
manifestIndex.clear();
|
|
342
371
|
listCache.clear();
|
|
372
|
+
invalidateListActive();
|
|
343
373
|
},
|
|
344
374
|
};
|
|
345
375
|
}
|
|
@@ -140,14 +140,15 @@ registerPlanTemplate({
|
|
|
140
140
|
{
|
|
141
141
|
name: "verify",
|
|
142
142
|
role: "verifier",
|
|
143
|
-
taskTemplate:
|
|
143
|
+
taskTemplate:
|
|
144
|
+
"Verify that all review findings are addressed. Run FAST checks (completes in <2 min): `npm run test:critical && npx tsc --noEmit`. Do NOT run `npm run test:unit` or `npm test` — too slow (642 files, >4 min). Confirm: {{goal}} is achieved.",
|
|
144
145
|
maxTasks: 1,
|
|
145
146
|
dependsOn: ["review"],
|
|
146
|
-
verificationCommand: "npm test",
|
|
147
|
+
verificationCommand: "npm run test:critical && npx tsc --noEmit",
|
|
147
148
|
},
|
|
148
149
|
],
|
|
149
150
|
verificationCommands: {
|
|
150
|
-
verify: "npm test",
|
|
151
|
+
verify: "npm run test:critical && npx tsc --noEmit",
|
|
151
152
|
},
|
|
152
153
|
});
|
|
153
154
|
|
|
@@ -187,13 +188,14 @@ registerPlanTemplate({
|
|
|
187
188
|
{
|
|
188
189
|
name: "verify",
|
|
189
190
|
role: "verifier",
|
|
190
|
-
taskTemplate:
|
|
191
|
+
taskTemplate:
|
|
192
|
+
"Verify the complete implementation of: {{goal}}. Run FAST checks (`npm run test:critical && npx tsc --noEmit`, completes in <2 min). Do NOT run `npm run test:unit` or `npm test` — too slow for in-loop verification (642 files, >4 min). Validate all acceptance criteria.",
|
|
191
193
|
maxTasks: 1,
|
|
192
194
|
dependsOn: ["review"],
|
|
193
|
-
verificationCommand: "npm test && npx tsc --noEmit",
|
|
195
|
+
verificationCommand: "npm run test:critical && npx tsc --noEmit",
|
|
194
196
|
},
|
|
195
197
|
],
|
|
196
198
|
verificationCommands: {
|
|
197
|
-
verify: "npm test && npx tsc --noEmit",
|
|
199
|
+
verify: "npm run test:critical && npx tsc --noEmit",
|
|
198
200
|
},
|
|
199
201
|
});
|
|
@@ -25,13 +25,11 @@
|
|
|
25
25
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
26
26
|
import { type ReplaceResult, replace } from "../runtime/replace.ts";
|
|
27
27
|
|
|
28
|
-
interface
|
|
28
|
+
interface EditToolLike {
|
|
29
29
|
name: string;
|
|
30
30
|
description: string;
|
|
31
31
|
parameters: unknown;
|
|
32
|
-
execute: (toolCallId: string, params:
|
|
33
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
34
|
-
[key: string]: any;
|
|
32
|
+
execute: (toolCallId: string, params: unknown, signal: unknown, onUpdate: unknown) => Promise<unknown>;
|
|
35
33
|
}
|
|
36
34
|
|
|
37
35
|
interface EditParams {
|
|
@@ -62,12 +60,11 @@ function isNotFoundResult(result: unknown): boolean {
|
|
|
62
60
|
/** Detect whether pi-diff is loaded (to avoid double-wrapping edit). */
|
|
63
61
|
function isPiDiffLoaded(pi: ExtensionAPI): boolean {
|
|
64
62
|
try {
|
|
65
|
-
|
|
66
|
-
const
|
|
67
|
-
const extensions = piAny?.extensions ?? piAny?._extensions ?? [];
|
|
63
|
+
const piExt = pi as unknown as { extensions?: unknown[] | Record<string, unknown>; _extensions?: unknown[] };
|
|
64
|
+
const extensions = piExt?.extensions ?? piExt?._extensions ?? [];
|
|
68
65
|
const names = Array.isArray(extensions)
|
|
69
66
|
? extensions.map((e: unknown) => (typeof e === "string" ? e : ((e as { name?: string })?.name ?? "")))
|
|
70
|
-
: Object.keys(extensions);
|
|
67
|
+
: Object.keys(extensions as Record<string, unknown>);
|
|
71
68
|
return names.some((n: string) => typeof n === "string" && n.includes("pi-diff"));
|
|
72
69
|
} catch {
|
|
73
70
|
return false;
|
|
@@ -82,28 +79,32 @@ function isPiDiffLoaded(pi: ExtensionAPI): boolean {
|
|
|
82
79
|
* @param tools optional injected tool registry (for testing)
|
|
83
80
|
* @returns true if the wrapper was applied, false if skipped
|
|
84
81
|
*/
|
|
85
|
-
export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit:
|
|
82
|
+
export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit: EditToolLike }): boolean {
|
|
86
83
|
// Auto-disable if pi-diff is present (it has its own replace integration).
|
|
87
84
|
if (isPiDiffLoaded(pi)) {
|
|
88
85
|
return false;
|
|
89
86
|
}
|
|
90
87
|
|
|
91
|
-
const t = tools ?? (pi as unknown as { tools?: { edit?:
|
|
88
|
+
const t = tools ?? (pi as unknown as { tools?: { edit?: EditToolLike } }).tools;
|
|
92
89
|
if (!t?.edit?.execute) return false;
|
|
93
90
|
|
|
94
91
|
const nativeExecute = t.edit.execute.bind(t.edit);
|
|
95
92
|
|
|
96
|
-
|
|
97
|
-
|
|
93
|
+
t.edit.execute = async function resilientExecute(
|
|
94
|
+
toolCallId: string,
|
|
95
|
+
params: unknown,
|
|
96
|
+
signal: unknown,
|
|
97
|
+
onUpdate: unknown,
|
|
98
|
+
): Promise<unknown> {
|
|
98
99
|
try {
|
|
99
100
|
const result = await nativeExecute(toolCallId, params, signal, onUpdate);
|
|
100
101
|
if (!isNotFoundResult(result)) return result;
|
|
101
102
|
// Fall through to resilient retry.
|
|
102
|
-
return await retryWithReplace(params, toolCallId, signal, onUpdate);
|
|
103
|
+
return await retryWithReplace(params as EditParams, toolCallId, signal, onUpdate);
|
|
103
104
|
} catch (err) {
|
|
104
105
|
const msg = err instanceof Error ? err.message : String(err);
|
|
105
106
|
if (NOT_FOUND_PATTERNS.some((re) => re.test(msg))) {
|
|
106
|
-
return await retryWithReplace(params, toolCallId, signal, onUpdate);
|
|
107
|
+
return await retryWithReplace(params as EditParams, toolCallId, signal, onUpdate);
|
|
107
108
|
}
|
|
108
109
|
throw err;
|
|
109
110
|
}
|
|
@@ -111,7 +112,7 @@ export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit: T
|
|
|
111
112
|
|
|
112
113
|
return true;
|
|
113
114
|
|
|
114
|
-
async function retryWithReplace(params: EditParams, toolCallId: string, signal:
|
|
115
|
+
async function retryWithReplace(params: EditParams, toolCallId: string, signal: unknown, onUpdate: unknown): Promise<unknown> {
|
|
115
116
|
const filePath = params.path ?? params.filePath;
|
|
116
117
|
const oldStr = params.oldString ?? params.old_string;
|
|
117
118
|
const newStr = params.newString ?? params.new_string;
|
|
@@ -1,20 +1,45 @@
|
|
|
1
1
|
export type RolePermissionMode = "read_only" | "workspace_write" | "danger_full_access" | "explicit_confirm";
|
|
2
2
|
|
|
3
|
+
// (FIND-12 R1 review: the AgentPermissionOptions opt-in was dropped — no caller
|
|
4
|
+
// passed `options` and AgentConfig has no `permissions` field, so it was dead
|
|
5
|
+
// code. Custom write-capable roles must instead be added to WRITE_ROLES below.)
|
|
6
|
+
|
|
3
7
|
// Read-only roles: cannot mutate files/source. `verifier` is NOT here — it runs
|
|
4
8
|
// tests (bash + cache writes) so it is a WRITE role (F4). `planner` stays
|
|
5
9
|
// read-only to preserve the plan-approval gate boundary (F3).
|
|
6
10
|
const READ_ONLY_ROLES = new Set(["explorer", "reviewer", "security-reviewer", "analyst", "critic", "planner"]);
|
|
7
|
-
|
|
11
|
+
// Write-capable roles: can mutate files/source within the workspace.
|
|
12
|
+
// FIND-12 (R1 review): this is an EXPLICIT allowlist — every shipped write-capable
|
|
13
|
+
// role must be listed here, otherwise default-deny demotes it to read-only.
|
|
14
|
+
// `agent` is the default direct-agent role (run.ts); `cold-verifier` runs bash/
|
|
15
|
+
// cache writes; `chain-executor` is the chain-workflow executor variant;
|
|
16
|
+
// `worker` is the autonomous goal-loop + dynamic-workflow executor role
|
|
17
|
+
// (goal-loop-runner.ts, run.ts dynamic workflow synthesis).
|
|
18
|
+
const WRITE_ROLES = new Set(["executor", "test-engineer", "writer", "verifier", "agent", "cold-verifier", "chain-executor", "worker"]);
|
|
8
19
|
export interface PermissionCheckResult {
|
|
9
20
|
allowed: boolean;
|
|
10
21
|
mode: RolePermissionMode;
|
|
11
22
|
reason?: string;
|
|
12
23
|
}
|
|
13
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the permission mode for a given role.
|
|
27
|
+
*
|
|
28
|
+
* FIND-12 (2026-07-20): **default-deny** — unknown/unrecognized roles now
|
|
29
|
+
* receive `"read_only"` instead of the previous permissive `"workspace_write"`.
|
|
30
|
+
* This prevents privilege escalation via typo'd or custom agent names that
|
|
31
|
+
* accidentally inherit write capabilities. Write-capable roles are an EXPLICIT
|
|
32
|
+
* allowlist (`WRITE_ROLES`); to add a custom write-capable role, add it there.
|
|
33
|
+
* (R1 review: a per-agent-config `permissions.workspaceWrite` opt-in was
|
|
34
|
+
* considered but dropped — it was unreachable from all 8 call sites.)
|
|
35
|
+
*
|
|
36
|
+
* @param role the role name (case-sensitive, must match exactly)
|
|
37
|
+
*/
|
|
14
38
|
export function permissionForRole(role: string): RolePermissionMode {
|
|
15
39
|
if (READ_ONLY_ROLES.has(role)) return "read_only";
|
|
16
40
|
if (WRITE_ROLES.has(role)) return "workspace_write";
|
|
17
|
-
|
|
41
|
+
// FIND-12: default-deny for unknown/unrecognized roles.
|
|
42
|
+
return "read_only";
|
|
18
43
|
}
|
|
19
44
|
|
|
20
45
|
export function currentCrewRole(env: NodeJS.ProcessEnv = process.env): string | undefined {
|
|
@@ -80,6 +80,18 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
80
80
|
|
|
81
81
|
let rawOutput = "";
|
|
82
82
|
let success = false;
|
|
83
|
+
// FIND-06: serialize heartbeat saves and retain the active save so terminal
|
|
84
|
+
// results can drain it before their final write.
|
|
85
|
+
let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
|
86
|
+
let heartbeatInFlight = false;
|
|
87
|
+
let heartbeatPromise: Promise<void> | null = null;
|
|
88
|
+
// FIND-06 P1 fix (R1 review): a heartbeat save that exceeds the 5s drain
|
|
89
|
+
// timeout continues in the background and can rename its temp file AFTER
|
|
90
|
+
// the terminal write, clobbering terminal state with the pre-terminal
|
|
91
|
+
// snapshot. finalWriteStarted lets the IIFE repair that by re-writing the
|
|
92
|
+
// (now-terminal) updatedTasks AFTER its own save resolves, so the terminal
|
|
93
|
+
// state always lands last regardless of disk timing.
|
|
94
|
+
let finalWriteStarted = false;
|
|
83
95
|
if (!executeWorkers) {
|
|
84
96
|
rawOutput = buildScaffoldOutput(groupTasks);
|
|
85
97
|
success = true;
|
|
@@ -87,20 +99,41 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
87
99
|
// Heartbeat refresher: touch every task's heartbeat every 15s while the
|
|
88
100
|
// worker is alive. Set `alive: true` explicitly so post-completion
|
|
89
101
|
// staleness checks immediately recognize liveness.
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
heartbeatTimer = setInterval(() => {
|
|
103
|
+
// FIND-06: setInterval does not await async callbacks. Do not start a
|
|
104
|
+
// second read/modify/write while the preceding heartbeat save is active.
|
|
105
|
+
if (heartbeatInFlight) return;
|
|
106
|
+
heartbeatInFlight = true;
|
|
107
|
+
heartbeatPromise = (async () => {
|
|
108
|
+
try {
|
|
109
|
+
updatedTasks = updatedTasks.map((t) => {
|
|
110
|
+
if (!taskIds.includes(t.id)) return t;
|
|
111
|
+
// FIND-06 belt-and-suspenders: never replace terminal state or its
|
|
112
|
+
// resultArtifact with a heartbeat-only snapshot.
|
|
113
|
+
if (t.status === "completed" || t.status === "failed") return t;
|
|
114
|
+
return {
|
|
115
|
+
...t,
|
|
116
|
+
heartbeat: touchWorkerHeartbeat(t.heartbeat ?? createWorkerHeartbeat(t.id), { alive: true }),
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
await saveRunTasksAsync(manifest, updatedTasks);
|
|
120
|
+
} catch {
|
|
121
|
+
// Run may have been pruned mid-dispatch — best-effort only.
|
|
122
|
+
} finally {
|
|
123
|
+
heartbeatInFlight = false;
|
|
124
|
+
// FIND-06 P1 fix: if the terminal write started while our heartbeat
|
|
125
|
+
// save was in flight, re-write the (now-terminal) updatedTasks so
|
|
126
|
+
// our late snapshot cannot leave stale pre-terminal state on disk.
|
|
127
|
+
// Runs after our own save resolves, so it always lands last.
|
|
128
|
+
if (finalWriteStarted) {
|
|
129
|
+
try {
|
|
130
|
+
await saveRunTasksAsync(manifest, updatedTasks);
|
|
131
|
+
} catch {
|
|
132
|
+
// best-effort repair — same swallow policy as the heartbeat.
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
})();
|
|
104
137
|
}, 15_000);
|
|
105
138
|
try {
|
|
106
139
|
const result = await executeWithRetry(
|
|
@@ -124,7 +157,7 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
124
157
|
rawOutput = `Worker dispatch failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
125
158
|
success = false;
|
|
126
159
|
} finally {
|
|
127
|
-
clearInterval(heartbeatTimer);
|
|
160
|
+
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
|
128
161
|
}
|
|
129
162
|
}
|
|
130
163
|
|
|
@@ -163,6 +196,30 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
163
196
|
resultArtifact,
|
|
164
197
|
};
|
|
165
198
|
});
|
|
199
|
+
|
|
200
|
+
// FIND-06: stop ticks and drain the snapshot captured by any active
|
|
201
|
+
// heartbeat before persisting terminal results. Bound the wait so a stuck
|
|
202
|
+
// filesystem operation cannot block dispatch completion indefinitely.
|
|
203
|
+
// P1 fix (R1 review): set finalWriteStarted BEFORE the drain so any pending
|
|
204
|
+
// heartbeat IIFE observes it and re-writes the terminal state after its
|
|
205
|
+
// own (possibly late) save resolves — preventing a late heartbeat snapshot
|
|
206
|
+
// from clobbering the terminal write.
|
|
207
|
+
finalWriteStarted = true;
|
|
208
|
+
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
|
209
|
+
const pendingHeartbeat = heartbeatPromise;
|
|
210
|
+
if (pendingHeartbeat) {
|
|
211
|
+
let drainTimeout: ReturnType<typeof setTimeout> | undefined;
|
|
212
|
+
try {
|
|
213
|
+
await Promise.race([
|
|
214
|
+
pendingHeartbeat,
|
|
215
|
+
new Promise<void>((resolve) => {
|
|
216
|
+
drainTimeout = setTimeout(resolve, 5_000);
|
|
217
|
+
}),
|
|
218
|
+
]);
|
|
219
|
+
} finally {
|
|
220
|
+
if (drainTimeout !== undefined) clearTimeout(drainTimeout);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
166
223
|
await saveRunTasksAsync(manifest, updatedTasks);
|
|
167
224
|
let updatedManifest: TeamRunManifest = {
|
|
168
225
|
...manifest,
|
|
@@ -85,7 +85,7 @@ export function buildTaskPacket(input: BuildTaskPacketInput): TaskPacket {
|
|
|
85
85
|
|
|
86
86
|
// Generate a deterministic hash-based task ID for traceability and logging.
|
|
87
87
|
// Uses goal + step ID + run ID as content parts.
|
|
88
|
-
// TODO:
|
|
88
|
+
// TODO(hashId): tracked — add hashId to TaskPacket when the schema supports it.
|
|
89
89
|
const _taskHashId = generateTaskHashId([input.manifest.goal, input.step.id, input.manifest.runId]);
|
|
90
90
|
|
|
91
91
|
return {
|
|
@@ -275,6 +275,19 @@ export const PiTeamsUiConfigSchema = Type.Object(
|
|
|
275
275
|
{ additionalProperties: false },
|
|
276
276
|
);
|
|
277
277
|
|
|
278
|
+
/** Phase 0 inter-pi broker config schema. Numeric limits are bounded per
|
|
279
|
+
* the plan: pathHashLen 4..32, maxFrameBytes 1024..1048576 (default 256 KiB),
|
|
280
|
+
* outboundQueueCap 32..4096 (default 256). */
|
|
281
|
+
export const CrewBrokerConfigSchema = Type.Object(
|
|
282
|
+
{
|
|
283
|
+
enabled: Type.Optional(Type.Boolean()),
|
|
284
|
+
pathHashLen: Type.Optional(Type.Integer({ minimum: 4, maximum: 32 })),
|
|
285
|
+
maxFrameBytes: Type.Optional(Type.Integer({ minimum: 1024, maximum: 1_048_576 })),
|
|
286
|
+
outboundQueueCap: Type.Optional(Type.Integer({ minimum: 32, maximum: 4096 })),
|
|
287
|
+
},
|
|
288
|
+
{ additionalProperties: false },
|
|
289
|
+
);
|
|
290
|
+
|
|
278
291
|
export const PiTeamsConfigSchema = Type.Object(
|
|
279
292
|
{
|
|
280
293
|
asyncByDefault: Type.Optional(Type.Boolean()),
|
|
@@ -297,6 +310,7 @@ export const PiTeamsConfigSchema = Type.Object(
|
|
|
297
310
|
reliability: Type.Optional(PiTeamsReliabilityConfigSchema),
|
|
298
311
|
otlp: Type.Optional(PiTeamsOtlpConfigSchema),
|
|
299
312
|
ui: Type.Optional(PiTeamsUiConfigSchema),
|
|
313
|
+
broker: Type.Optional(CrewBrokerConfigSchema),
|
|
300
314
|
},
|
|
301
315
|
{ additionalProperties: false },
|
|
302
316
|
);
|
package/src/state/event-log.ts
CHANGED
|
@@ -4,7 +4,7 @@ import * as path from "node:path";
|
|
|
4
4
|
import { DEFAULT_EVENT_LOG } from "../config/defaults.ts";
|
|
5
5
|
import { errors } from "../errors.ts";
|
|
6
6
|
import { emitFromTeamEvent } from "../ui/run-event-bus.ts";
|
|
7
|
-
import { type IncrementalReadState, readJsonlSince } from "../utils/incremental-reader.ts";
|
|
7
|
+
import { type IncrementalReadState, readJsonlSince, readJsonlTail } from "../utils/incremental-reader.ts";
|
|
8
8
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
9
9
|
import { redactSecrets } from "../utils/redaction.ts";
|
|
10
10
|
import { sleepSync } from "../utils/sleep.ts";
|
|
@@ -543,9 +543,13 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
543
543
|
} catch {
|
|
544
544
|
/* file does not exist */
|
|
545
545
|
}
|
|
546
|
+
// FIND-10: track whether overflow handling modified the file so we can
|
|
547
|
+
// reuse fileStat for the post-overflow size check (avoids redundant stat).
|
|
548
|
+
let overflowHandled = false;
|
|
546
549
|
if (!isTerminal && fileStat) {
|
|
547
550
|
const stat = fileStat;
|
|
548
551
|
if (stat.size > MAX_EVENTS_BYTES) {
|
|
552
|
+
overflowHandled = true;
|
|
549
553
|
try {
|
|
550
554
|
compactEventLog(eventsPath);
|
|
551
555
|
} catch (error) {
|
|
@@ -564,11 +568,17 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
564
568
|
}
|
|
565
569
|
}
|
|
566
570
|
}
|
|
571
|
+
// FIND-10: collapse redundant stat. If no overflow handling occurred,
|
|
572
|
+
// the file hasn't changed since fileStat — reuse it instead of re-stat'ing.
|
|
567
573
|
let sizeCheckStat: fs.Stats | undefined;
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
574
|
+
if (overflowHandled) {
|
|
575
|
+
try {
|
|
576
|
+
sizeCheckStat = await fs.promises.stat(eventsPath).catch(() => undefined);
|
|
577
|
+
} catch {
|
|
578
|
+
/* file does not exist */
|
|
579
|
+
}
|
|
580
|
+
} else {
|
|
581
|
+
sizeCheckStat = fileStat;
|
|
572
582
|
}
|
|
573
583
|
try {
|
|
574
584
|
if (sizeCheckStat && sizeCheckStat.size > MAX_EVENTS_BYTES) {
|
|
@@ -583,25 +593,44 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
583
593
|
logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
|
|
584
594
|
}
|
|
585
595
|
|
|
596
|
+
// FIND-10: post-append stat captured from the same fd (non-worker path)
|
|
597
|
+
// for reuse in the cache update below, avoiding a redundant path stat.
|
|
598
|
+
let postAppendStat: fs.Stats | undefined;
|
|
586
599
|
if (!skippedDueToSize) {
|
|
587
600
|
const line = JSON.stringify(redactSecrets(fullEvent)) + "\n";
|
|
588
601
|
// Phase 1.5: when worker atomic writer is enabled, append via worker.
|
|
589
602
|
if (isWorkerAtomicWriterEnabled()) {
|
|
590
603
|
await appendFileViaWorker(eventsPath, line);
|
|
604
|
+
// Worker path: fsync via a separate open (worker manages its own fd).
|
|
605
|
+
const fd = await fs.promises.open(eventsPath, "r+");
|
|
606
|
+
try {
|
|
607
|
+
await fd.sync();
|
|
608
|
+
} finally {
|
|
609
|
+
await fd.close();
|
|
610
|
+
}
|
|
591
611
|
} else {
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
612
|
+
// FIND-10: single-fd append+fsync. Opens in append mode, writes,
|
|
613
|
+
// fsyncs on the SAME fd, then closes — eliminating the separate
|
|
614
|
+
// open("r+") + sync that previously doubled the fd count. The
|
|
615
|
+
// fsync (seq-integrity protection) is preserved exactly: it still
|
|
616
|
+
// closes the crash window between append and persistSequence.
|
|
617
|
+
const fd = await fs.promises.open(eventsPath, "a");
|
|
618
|
+
try {
|
|
619
|
+
await fd.appendFile(line, "utf-8");
|
|
620
|
+
await fd.sync();
|
|
621
|
+
// FIND-10 R1 fix: the cache-optimization fd.stat() must NOT sit in the
|
|
622
|
+
// seq-durability critical path. If it threw (rare — fd invalidated),
|
|
623
|
+
// it would skip persistSequence below and reopen the seq-reuse
|
|
624
|
+
// window the fsync just closed. Guard it; fall back to undefined
|
|
625
|
+
// (the later cache-update takes a path stat instead).
|
|
626
|
+
try {
|
|
627
|
+
postAppendStat = await fd.stat();
|
|
628
|
+
} catch {
|
|
629
|
+
postAppendStat = undefined;
|
|
630
|
+
}
|
|
631
|
+
} finally {
|
|
632
|
+
await fd.close();
|
|
633
|
+
}
|
|
605
634
|
}
|
|
606
635
|
// FIX: Persist sequence AFTER successful appendFile to ensure sidecar
|
|
607
636
|
// is only updated when the event is definitively written. If appendFile
|
|
@@ -609,7 +638,11 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
609
638
|
// preventing sequence reuse on restart.
|
|
610
639
|
persistSequence(eventsPath, seq);
|
|
611
640
|
}
|
|
641
|
+
// FIND-10: track whether compaction happened after the append so the
|
|
642
|
+
// cache-update stat can safely reuse postAppendStat (file unchanged).
|
|
643
|
+
let compactedAfterAppend = false;
|
|
612
644
|
if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
|
|
645
|
+
compactedAfterAppend = true;
|
|
613
646
|
try {
|
|
614
647
|
compactEventLog(eventsPath);
|
|
615
648
|
} catch (error) {
|
|
@@ -626,11 +659,18 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
626
659
|
// Only update the cache here (the sidecar persist is already done).
|
|
627
660
|
const finalSeq = fullEvent.metadata?.seq ?? 0;
|
|
628
661
|
try {
|
|
662
|
+
// FIND-10: reuse post-append fd stat when available and no compaction
|
|
663
|
+
// happened after the append (file unchanged). Falls back to path stat
|
|
664
|
+
// for the worker path, skipped events, or post-compaction cases.
|
|
629
665
|
let statResult: fs.Stats | undefined;
|
|
630
|
-
|
|
631
|
-
statResult =
|
|
632
|
-
}
|
|
633
|
-
|
|
666
|
+
if (postAppendStat && !compactedAfterAppend) {
|
|
667
|
+
statResult = postAppendStat;
|
|
668
|
+
} else {
|
|
669
|
+
try {
|
|
670
|
+
statResult = await fs.promises.stat(eventsPath).catch(() => undefined);
|
|
671
|
+
} catch {
|
|
672
|
+
/* file may not exist */
|
|
673
|
+
}
|
|
634
674
|
}
|
|
635
675
|
if (statResult) {
|
|
636
676
|
if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
|
|
@@ -1217,24 +1257,38 @@ export function readEventsCursor(eventsPath: string, options: EventCursorOptions
|
|
|
1217
1257
|
};
|
|
1218
1258
|
}
|
|
1219
1259
|
|
|
1220
|
-
//
|
|
1221
|
-
//
|
|
1222
|
-
//
|
|
1223
|
-
//
|
|
1224
|
-
//
|
|
1260
|
+
// FIND-05 default path: byte-level tail read (last 4MB) instead of
|
|
1261
|
+
// full-file read. Bounds CPU to O(tail bytes) instead of O(total
|
|
1262
|
+
// events). The legacy readEvents() full parse path is preserved for
|
|
1263
|
+
// callers that explicitly need the full history (e.g. tests that
|
|
1264
|
+
// assert exact contents) and as a small-file fallback.
|
|
1265
|
+
//
|
|
1266
|
+
// The 5000-event tail cap and the "event-log.cursor-full-read"
|
|
1267
|
+
// warning are preserved. A separate cursor-tail-truncated warning is
|
|
1268
|
+
// emitted whenever the file exceeds the 4MB tail budget, signalling
|
|
1269
|
+
// that a prefix was dropped and callers should pass fromByteOffset for
|
|
1270
|
+
// streaming reads.
|
|
1271
|
+
const TAIL_BYTES = 4 * 1024 * 1024; // 4 MB
|
|
1272
|
+
const TAIL_EVENT_CAP = 5000;
|
|
1225
1273
|
const sinceSeq = positiveInteger(options.sinceSeq) ?? 0;
|
|
1226
1274
|
const limit = positiveInteger(options.limit);
|
|
1227
|
-
|
|
1228
|
-
const
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1275
|
+
|
|
1276
|
+
const tail = readJsonlTail<TeamEvent>(eventsPath, TAIL_BYTES);
|
|
1277
|
+
let all = tail.items;
|
|
1278
|
+
if (tail.truncated) {
|
|
1279
|
+
logInternalError("event-log.cursor-tail-truncated", {
|
|
1280
|
+
eventsPath,
|
|
1281
|
+
returned: all.length,
|
|
1282
|
+
tailBytes: TAIL_BYTES,
|
|
1283
|
+
});
|
|
1284
|
+
}
|
|
1285
|
+
if (all.length > TAIL_EVENT_CAP) {
|
|
1232
1286
|
logInternalError(
|
|
1233
1287
|
"event-log.cursor-full-read",
|
|
1234
|
-
new Error(`readEventsCursor read
|
|
1288
|
+
new Error(`readEventsCursor tail read dropped events from a larger log; pass fromByteOffset for incremental reads`),
|
|
1235
1289
|
`eventsPath=${eventsPath}`,
|
|
1236
1290
|
);
|
|
1237
|
-
all = all.slice(-
|
|
1291
|
+
all = all.slice(-TAIL_EVENT_CAP);
|
|
1238
1292
|
}
|
|
1239
1293
|
const filtered = all.filter((event) => (event.metadata?.seq ?? 0) > sinceSeq);
|
|
1240
1294
|
const events = limit !== undefined ? filtered.slice(0, limit) : filtered;
|