pi-crew 0.9.42 → 0.9.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +223 -0
- package/README.md +35 -0
- package/dist/build-meta.json +4778 -11087
- package/dist/index.mjs +54648 -65435
- package/dist/index.mjs.map +4 -4
- package/package.json +1 -1
- package/scripts/build-bundle.mjs +2 -1
- package/src/agents/agent-config.ts +1 -1
- package/src/agents/discover-agents.ts +1 -1
- package/src/config/config.ts +1 -1
- package/src/config/types.ts +0 -2
- package/src/extension/crew-vibes/config.ts +1 -1
- package/src/extension/crew-vibes/font-detect.ts +16 -3
- package/src/extension/cross-extension-rpc.ts +8 -4
- package/src/extension/registration/commands.ts +5 -1
- package/src/extension/registration/foreground-run-controller.ts +28 -0
- package/src/extension/registration/lifecycle-handlers.ts +34 -3
- package/src/extension/registration/subagent-manager-setup.ts +178 -59
- package/src/extension/run-import.ts +21 -1
- package/src/extension/team-tool/api.ts +4 -2
- package/src/extension/team-tool.ts +5 -0
- package/src/prompt/prompt-runtime.ts +15 -0
- package/src/runtime/async-runner.ts +9 -1
- package/src/runtime/child-pi-constants.ts +42 -0
- package/src/runtime/child-pi-kill.ts +180 -0
- package/src/runtime/child-pi-spawn.ts +234 -0
- package/src/runtime/child-pi-steering.ts +128 -0
- package/src/runtime/child-pi-streams.ts +296 -0
- package/src/runtime/child-pi-transcript.ts +169 -0
- package/src/runtime/child-pi.ts +75 -837
- package/src/runtime/compact-stages/tail-capture-stage.ts +1 -1
- package/src/runtime/dwf-state-store.ts +5 -0
- package/src/runtime/dynamic-workflow-context.ts +78 -18
- package/src/runtime/dynamic-workflow-runner.ts +18 -2
- package/src/runtime/goal-evaluator.ts +59 -27
- package/src/runtime/goal-loop-runner.ts +40 -11
- package/src/runtime/live-agent-manager.ts +18 -0
- package/src/runtime/manifest-cache.ts +30 -0
- package/src/runtime/pi-json-output.ts +2 -1
- package/src/runtime/pi-spawn.ts +7 -1
- package/src/runtime/resilient-edit.ts +16 -15
- package/src/runtime/role-permission.ts +27 -2
- package/src/runtime/run-coalesced-task-group.ts +90 -25
- package/src/runtime/task-packet.ts +1 -1
- package/src/runtime/task-runner/prompt-builder.ts +2 -2
- package/src/runtime/team-runner.ts +6 -1
- package/src/state/event-log.ts +89 -34
- package/src/state/locks.ts +185 -49
- package/src/state/mailbox.ts +165 -4
- package/src/state/run-metrics.ts +40 -12
- package/src/state/worker-atomic-writer.ts +1 -0
- package/src/ui/live-run-sidebar.ts +1 -9
- package/src/ui/run-dashboard.ts +1 -9
- package/src/utils/incremental-reader.ts +105 -0
- package/src/utils/paths.ts +1 -1
- package/src/utils/visual.ts +27 -91
- package/src/worktree/worktree-manager.ts +47 -19
- package/src/runtime/auto-resume.ts +0 -100
- package/src/runtime/notebook-helpers.ts +0 -88
- package/src/runtime/orphan-sentinel.ts +0 -7
|
@@ -25,13 +25,11 @@
|
|
|
25
25
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
26
26
|
import { type ReplaceResult, replace } from "../runtime/replace.ts";
|
|
27
27
|
|
|
28
|
-
interface
|
|
28
|
+
interface EditToolLike {
|
|
29
29
|
name: string;
|
|
30
30
|
description: string;
|
|
31
31
|
parameters: unknown;
|
|
32
|
-
execute: (toolCallId: string, params:
|
|
33
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
34
|
-
[key: string]: any;
|
|
32
|
+
execute: (toolCallId: string, params: unknown, signal: unknown, onUpdate: unknown) => Promise<unknown>;
|
|
35
33
|
}
|
|
36
34
|
|
|
37
35
|
interface EditParams {
|
|
@@ -62,12 +60,11 @@ function isNotFoundResult(result: unknown): boolean {
|
|
|
62
60
|
/** Detect whether pi-diff is loaded (to avoid double-wrapping edit). */
|
|
63
61
|
function isPiDiffLoaded(pi: ExtensionAPI): boolean {
|
|
64
62
|
try {
|
|
65
|
-
|
|
66
|
-
const
|
|
67
|
-
const extensions = piAny?.extensions ?? piAny?._extensions ?? [];
|
|
63
|
+
const piExt = pi as unknown as { extensions?: unknown[] | Record<string, unknown>; _extensions?: unknown[] };
|
|
64
|
+
const extensions = piExt?.extensions ?? piExt?._extensions ?? [];
|
|
68
65
|
const names = Array.isArray(extensions)
|
|
69
66
|
? extensions.map((e: unknown) => (typeof e === "string" ? e : ((e as { name?: string })?.name ?? "")))
|
|
70
|
-
: Object.keys(extensions);
|
|
67
|
+
: Object.keys(extensions as Record<string, unknown>);
|
|
71
68
|
return names.some((n: string) => typeof n === "string" && n.includes("pi-diff"));
|
|
72
69
|
} catch {
|
|
73
70
|
return false;
|
|
@@ -82,28 +79,32 @@ function isPiDiffLoaded(pi: ExtensionAPI): boolean {
|
|
|
82
79
|
* @param tools optional injected tool registry (for testing)
|
|
83
80
|
* @returns true if the wrapper was applied, false if skipped
|
|
84
81
|
*/
|
|
85
|
-
export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit:
|
|
82
|
+
export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit: EditToolLike }): boolean {
|
|
86
83
|
// Auto-disable if pi-diff is present (it has its own replace integration).
|
|
87
84
|
if (isPiDiffLoaded(pi)) {
|
|
88
85
|
return false;
|
|
89
86
|
}
|
|
90
87
|
|
|
91
|
-
const t = tools ?? (pi as unknown as { tools?: { edit?:
|
|
88
|
+
const t = tools ?? (pi as unknown as { tools?: { edit?: EditToolLike } }).tools;
|
|
92
89
|
if (!t?.edit?.execute) return false;
|
|
93
90
|
|
|
94
91
|
const nativeExecute = t.edit.execute.bind(t.edit);
|
|
95
92
|
|
|
96
|
-
|
|
97
|
-
|
|
93
|
+
t.edit.execute = async function resilientExecute(
|
|
94
|
+
toolCallId: string,
|
|
95
|
+
params: unknown,
|
|
96
|
+
signal: unknown,
|
|
97
|
+
onUpdate: unknown,
|
|
98
|
+
): Promise<unknown> {
|
|
98
99
|
try {
|
|
99
100
|
const result = await nativeExecute(toolCallId, params, signal, onUpdate);
|
|
100
101
|
if (!isNotFoundResult(result)) return result;
|
|
101
102
|
// Fall through to resilient retry.
|
|
102
|
-
return await retryWithReplace(params, toolCallId, signal, onUpdate);
|
|
103
|
+
return await retryWithReplace(params as EditParams, toolCallId, signal, onUpdate);
|
|
103
104
|
} catch (err) {
|
|
104
105
|
const msg = err instanceof Error ? err.message : String(err);
|
|
105
106
|
if (NOT_FOUND_PATTERNS.some((re) => re.test(msg))) {
|
|
106
|
-
return await retryWithReplace(params, toolCallId, signal, onUpdate);
|
|
107
|
+
return await retryWithReplace(params as EditParams, toolCallId, signal, onUpdate);
|
|
107
108
|
}
|
|
108
109
|
throw err;
|
|
109
110
|
}
|
|
@@ -111,7 +112,7 @@ export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit: T
|
|
|
111
112
|
|
|
112
113
|
return true;
|
|
113
114
|
|
|
114
|
-
async function retryWithReplace(params: EditParams, toolCallId: string, signal:
|
|
115
|
+
async function retryWithReplace(params: EditParams, toolCallId: string, signal: unknown, onUpdate: unknown): Promise<unknown> {
|
|
115
116
|
const filePath = params.path ?? params.filePath;
|
|
116
117
|
const oldStr = params.oldString ?? params.old_string;
|
|
117
118
|
const newStr = params.newString ?? params.new_string;
|
|
@@ -1,20 +1,45 @@
|
|
|
1
1
|
export type RolePermissionMode = "read_only" | "workspace_write" | "danger_full_access" | "explicit_confirm";
|
|
2
2
|
|
|
3
|
+
// (FIND-12 R1 review: the AgentPermissionOptions opt-in was dropped — no caller
|
|
4
|
+
// passed `options` and AgentConfig has no `permissions` field, so it was dead
|
|
5
|
+
// code. Custom write-capable roles must instead be added to WRITE_ROLES below.)
|
|
6
|
+
|
|
3
7
|
// Read-only roles: cannot mutate files/source. `verifier` is NOT here — it runs
|
|
4
8
|
// tests (bash + cache writes) so it is a WRITE role (F4). `planner` stays
|
|
5
9
|
// read-only to preserve the plan-approval gate boundary (F3).
|
|
6
10
|
const READ_ONLY_ROLES = new Set(["explorer", "reviewer", "security-reviewer", "analyst", "critic", "planner"]);
|
|
7
|
-
|
|
11
|
+
// Write-capable roles: can mutate files/source within the workspace.
|
|
12
|
+
// FIND-12 (R1 review): this is an EXPLICIT allowlist — every shipped write-capable
|
|
13
|
+
// role must be listed here, otherwise default-deny demotes it to read-only.
|
|
14
|
+
// `agent` is the default direct-agent role (run.ts); `cold-verifier` runs bash/
|
|
15
|
+
// cache writes; `chain-executor` is the chain-workflow executor variant;
|
|
16
|
+
// `worker` is the autonomous goal-loop + dynamic-workflow executor role
|
|
17
|
+
// (goal-loop-runner.ts, run.ts dynamic workflow synthesis).
|
|
18
|
+
const WRITE_ROLES = new Set(["executor", "test-engineer", "writer", "verifier", "agent", "cold-verifier", "chain-executor", "worker"]);
|
|
8
19
|
export interface PermissionCheckResult {
|
|
9
20
|
allowed: boolean;
|
|
10
21
|
mode: RolePermissionMode;
|
|
11
22
|
reason?: string;
|
|
12
23
|
}
|
|
13
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the permission mode for a given role.
|
|
27
|
+
*
|
|
28
|
+
* FIND-12 (2026-07-20): **default-deny** — unknown/unrecognized roles now
|
|
29
|
+
* receive `"read_only"` instead of the previous permissive `"workspace_write"`.
|
|
30
|
+
* This prevents privilege escalation via typo'd or custom agent names that
|
|
31
|
+
* accidentally inherit write capabilities. Write-capable roles are an EXPLICIT
|
|
32
|
+
* allowlist (`WRITE_ROLES`); to add a custom write-capable role, add it there.
|
|
33
|
+
* (R1 review: a per-agent-config `permissions.workspaceWrite` opt-in was
|
|
34
|
+
* considered but dropped — it was unreachable from all 8 call sites.)
|
|
35
|
+
*
|
|
36
|
+
* @param role the role name (case-sensitive, must match exactly)
|
|
37
|
+
*/
|
|
14
38
|
export function permissionForRole(role: string): RolePermissionMode {
|
|
15
39
|
if (READ_ONLY_ROLES.has(role)) return "read_only";
|
|
16
40
|
if (WRITE_ROLES.has(role)) return "workspace_write";
|
|
17
|
-
|
|
41
|
+
// FIND-12: default-deny for unknown/unrecognized roles.
|
|
42
|
+
return "read_only";
|
|
18
43
|
}
|
|
19
44
|
|
|
20
45
|
export function currentCrewRole(env: NodeJS.ProcessEnv = process.env): string | undefined {
|
|
@@ -5,8 +5,10 @@ import { saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
|
|
|
5
5
|
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
6
6
|
import type { WorkflowStep } from "../workflows/workflow-config.ts";
|
|
7
7
|
import { runChildPi } from "./child-pi.ts";
|
|
8
|
+
import { DEFAULT_RETRY_POLICY, executeWithRetry } from "./retry-executor.ts";
|
|
8
9
|
import { permissionForRole } from "./role-permission.ts";
|
|
9
10
|
import type { CrewRuntimeMode } from "./runtime-resolver.ts";
|
|
11
|
+
import { sanitizeTaskText } from "./task-packet.ts";
|
|
10
12
|
import { splitCoalescedOutput } from "./task-runner/output-splitter.ts";
|
|
11
13
|
import { mergeArtifacts } from "./team-runner-artifacts.ts";
|
|
12
14
|
import { createWorkerHeartbeat, touchWorkerHeartbeat } from "./worker-heartbeat.ts";
|
|
@@ -78,6 +80,18 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
78
80
|
|
|
79
81
|
let rawOutput = "";
|
|
80
82
|
let success = false;
|
|
83
|
+
// FIND-06: serialize heartbeat saves and retain the active save so terminal
|
|
84
|
+
// results can drain it before their final write.
|
|
85
|
+
let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
|
86
|
+
let heartbeatInFlight = false;
|
|
87
|
+
let heartbeatPromise: Promise<void> | null = null;
|
|
88
|
+
// FIND-06 P1 fix (R1 review): a heartbeat save that exceeds the 5s drain
|
|
89
|
+
// timeout continues in the background and can rename its temp file AFTER
|
|
90
|
+
// the terminal write, clobbering terminal state with the pre-terminal
|
|
91
|
+
// snapshot. finalWriteStarted lets the IIFE repair that by re-writing the
|
|
92
|
+
// (now-terminal) updatedTasks AFTER its own save resolves, so the terminal
|
|
93
|
+
// state always lands last regardless of disk timing.
|
|
94
|
+
let finalWriteStarted = false;
|
|
81
95
|
if (!executeWorkers) {
|
|
82
96
|
rawOutput = buildScaffoldOutput(groupTasks);
|
|
83
97
|
success = true;
|
|
@@ -85,38 +99,65 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
85
99
|
// Heartbeat refresher: touch every task's heartbeat every 15s while the
|
|
86
100
|
// worker is alive. Set `alive: true` explicitly so post-completion
|
|
87
101
|
// staleness checks immediately recognize liveness.
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
+
heartbeatTimer = setInterval(() => {
|
|
103
|
+
// FIND-06: setInterval does not await async callbacks. Do not start a
|
|
104
|
+
// second read/modify/write while the preceding heartbeat save is active.
|
|
105
|
+
if (heartbeatInFlight) return;
|
|
106
|
+
heartbeatInFlight = true;
|
|
107
|
+
heartbeatPromise = (async () => {
|
|
108
|
+
try {
|
|
109
|
+
updatedTasks = updatedTasks.map((t) => {
|
|
110
|
+
if (!taskIds.includes(t.id)) return t;
|
|
111
|
+
// FIND-06 belt-and-suspenders: never replace terminal state or its
|
|
112
|
+
// resultArtifact with a heartbeat-only snapshot.
|
|
113
|
+
if (t.status === "completed" || t.status === "failed") return t;
|
|
114
|
+
return {
|
|
115
|
+
...t,
|
|
116
|
+
heartbeat: touchWorkerHeartbeat(t.heartbeat ?? createWorkerHeartbeat(t.id), { alive: true }),
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
await saveRunTasksAsync(manifest, updatedTasks);
|
|
120
|
+
} catch {
|
|
121
|
+
// Run may have been pruned mid-dispatch — best-effort only.
|
|
122
|
+
} finally {
|
|
123
|
+
heartbeatInFlight = false;
|
|
124
|
+
// FIND-06 P1 fix: if the terminal write started while our heartbeat
|
|
125
|
+
// save was in flight, re-write the (now-terminal) updatedTasks so
|
|
126
|
+
// our late snapshot cannot leave stale pre-terminal state on disk.
|
|
127
|
+
// Runs after our own save resolves, so it always lands last.
|
|
128
|
+
if (finalWriteStarted) {
|
|
129
|
+
try {
|
|
130
|
+
await saveRunTasksAsync(manifest, updatedTasks);
|
|
131
|
+
} catch {
|
|
132
|
+
// best-effort repair — same swallow policy as the heartbeat.
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
})();
|
|
102
137
|
}, 15_000);
|
|
103
138
|
try {
|
|
104
|
-
const result = await
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
139
|
+
const result = await executeWithRetry(
|
|
140
|
+
async () => {
|
|
141
|
+
return await runChildPi({
|
|
142
|
+
cwd: firstTask.cwd,
|
|
143
|
+
task: combinedPrompt,
|
|
144
|
+
agent,
|
|
145
|
+
signal,
|
|
146
|
+
excludeContextBash: true,
|
|
147
|
+
maxTurns: 5,
|
|
148
|
+
onJsonEvent: (e) => input.onJsonEvent?.(firstTask.id, manifest.runId, e),
|
|
149
|
+
});
|
|
150
|
+
},
|
|
151
|
+
DEFAULT_RETRY_POLICY,
|
|
152
|
+
{ signal },
|
|
153
|
+
);
|
|
113
154
|
rawOutput = result.rawFinalText ?? result.stdout ?? "";
|
|
114
155
|
success = result.exitStatus?.exitCode === 0;
|
|
115
156
|
} catch (err) {
|
|
116
157
|
rawOutput = `Worker dispatch failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
117
158
|
success = false;
|
|
118
159
|
} finally {
|
|
119
|
-
clearInterval(heartbeatTimer);
|
|
160
|
+
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
|
120
161
|
}
|
|
121
162
|
}
|
|
122
163
|
|
|
@@ -155,6 +196,30 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
155
196
|
resultArtifact,
|
|
156
197
|
};
|
|
157
198
|
});
|
|
199
|
+
|
|
200
|
+
// FIND-06: stop ticks and drain the snapshot captured by any active
|
|
201
|
+
// heartbeat before persisting terminal results. Bound the wait so a stuck
|
|
202
|
+
// filesystem operation cannot block dispatch completion indefinitely.
|
|
203
|
+
// P1 fix (R1 review): set finalWriteStarted BEFORE the drain so any pending
|
|
204
|
+
// heartbeat IIFE observes it and re-writes the terminal state after its
|
|
205
|
+
// own (possibly late) save resolves — preventing a late heartbeat snapshot
|
|
206
|
+
// from clobbering the terminal write.
|
|
207
|
+
finalWriteStarted = true;
|
|
208
|
+
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
|
209
|
+
const pendingHeartbeat = heartbeatPromise;
|
|
210
|
+
if (pendingHeartbeat) {
|
|
211
|
+
let drainTimeout: ReturnType<typeof setTimeout> | undefined;
|
|
212
|
+
try {
|
|
213
|
+
await Promise.race([
|
|
214
|
+
pendingHeartbeat,
|
|
215
|
+
new Promise<void>((resolve) => {
|
|
216
|
+
drainTimeout = setTimeout(resolve, 5_000);
|
|
217
|
+
}),
|
|
218
|
+
]);
|
|
219
|
+
} finally {
|
|
220
|
+
if (drainTimeout !== undefined) clearTimeout(drainTimeout);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
158
223
|
await saveRunTasksAsync(manifest, updatedTasks);
|
|
159
224
|
let updatedManifest: TeamRunManifest = {
|
|
160
225
|
...manifest,
|
|
@@ -193,7 +258,7 @@ async function buildCoalescedPrompt(
|
|
|
193
258
|
`### Task ${idx + 1} of ${groupTasks.length} (id: ${task.id})`,
|
|
194
259
|
`Step: ${step.id}`,
|
|
195
260
|
`Role: ${step.role}`,
|
|
196
|
-
`Task: ${step.task.replaceAll("{goal}", manifest.goal)}`,
|
|
261
|
+
`Task: ${sanitizeTaskText(step.task.replaceAll("{goal}", manifest.goal))}`,
|
|
197
262
|
].join("\n");
|
|
198
263
|
})
|
|
199
264
|
.join("\n\n---\n\n");
|
|
@@ -85,7 +85,7 @@ export function buildTaskPacket(input: BuildTaskPacketInput): TaskPacket {
|
|
|
85
85
|
|
|
86
86
|
// Generate a deterministic hash-based task ID for traceability and logging.
|
|
87
87
|
// Uses goal + step ID + run ID as content parts.
|
|
88
|
-
// TODO:
|
|
88
|
+
// TODO(hashId): tracked — add hashId to TaskPacket when the schema supports it.
|
|
89
89
|
const _taskHashId = generateTaskHashId([input.manifest.goal, input.step.id, input.manifest.runId]);
|
|
90
90
|
|
|
91
91
|
return {
|
|
@@ -4,7 +4,7 @@ import type { TaskOutputSchema, TeamRunManifest, TeamTaskState } from "../../sta
|
|
|
4
4
|
import type { WorkflowStep } from "../../workflows/workflow-config.ts";
|
|
5
5
|
import { buildMemoryBlock } from "../agent-memory.ts";
|
|
6
6
|
import { permissionForRole } from "../role-permission.ts";
|
|
7
|
-
import { HANDOFF_TEMPLATE, renderTaskPacket } from "../task-packet.ts";
|
|
7
|
+
import { HANDOFF_TEMPLATE, renderTaskPacket, sanitizeTaskText } from "../task-packet.ts";
|
|
8
8
|
import { buildWorkspaceTree } from "../workspace-tree.ts";
|
|
9
9
|
import { renderSuggestedFilesSection, runRetrievalCycle } from "./retrieval-orchestrator.ts";
|
|
10
10
|
|
|
@@ -267,7 +267,7 @@ export async function renderTaskPrompt(
|
|
|
267
267
|
memoryBlock,
|
|
268
268
|
task.taskPacket?.outputSchema ? renderOutputSchemaBlock(task.taskPacket.outputSchema) : "",
|
|
269
269
|
"Task:",
|
|
270
|
-
step.task.replaceAll("{goal}", manifest.goal),
|
|
270
|
+
sanitizeTaskText(step.task.replaceAll("{goal}", manifest.goal)),
|
|
271
271
|
"",
|
|
272
272
|
"When your task is complete, structure your final output using this handoff template:",
|
|
273
273
|
HANDOFF_TEMPLATE,
|
|
@@ -9,7 +9,7 @@ import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
|
9
9
|
import { PluginRegistry } from "../plugins/plugin-registry.ts";
|
|
10
10
|
import { NextJsPlugin, VitePlugin, VitestPlugin } from "../plugins/plugins/index.ts";
|
|
11
11
|
import { hashArtifactContent as hashContent, writeArtifact } from "../state/artifact-store.ts";
|
|
12
|
-
import { atomicWriteFile } from "../state/atomic-write.ts";
|
|
12
|
+
import { atomicWriteFile, flushPendingAtomicWrites } from "../state/atomic-write.ts";
|
|
13
13
|
import { appendEvent, appendEventAsync, appendEventBuffered, appendEventFireAndForget, flushEventLogBuffer } from "../state/event-log.ts";
|
|
14
14
|
import { HealthStore } from "../state/health-store.ts";
|
|
15
15
|
import { withRunLock } from "../state/locks.ts";
|
|
@@ -1615,6 +1615,11 @@ async function executeTeamRunCore(
|
|
|
1615
1615
|
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
1616
1616
|
// on committed state, not in-memory state that may differ from disk.
|
|
1617
1617
|
const mergeResult = await withRunLock(manifest, async () => {
|
|
1618
|
+
// NEW-D1: flush any pending coalesced atomic writes before reading from
|
|
1619
|
+
// disk. Without this, a worker's async manifest save (coalesced by
|
|
1620
|
+
// atomic-write) may not be committed yet, causing a lost-update on the
|
|
1621
|
+
// merge read. flushPendingAtomicWrites forces all queued writes to disk.
|
|
1622
|
+
flushPendingAtomicWrites();
|
|
1618
1623
|
const disk = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
1619
1624
|
const diskManifest = disk?.manifest ?? manifest;
|
|
1620
1625
|
const diskArtifacts = diskManifest.artifacts;
|
package/src/state/event-log.ts
CHANGED
|
@@ -4,7 +4,7 @@ import * as path from "node:path";
|
|
|
4
4
|
import { DEFAULT_EVENT_LOG } from "../config/defaults.ts";
|
|
5
5
|
import { errors } from "../errors.ts";
|
|
6
6
|
import { emitFromTeamEvent } from "../ui/run-event-bus.ts";
|
|
7
|
-
import { type IncrementalReadState, readJsonlSince } from "../utils/incremental-reader.ts";
|
|
7
|
+
import { type IncrementalReadState, readJsonlSince, readJsonlTail } from "../utils/incremental-reader.ts";
|
|
8
8
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
9
9
|
import { redactSecrets } from "../utils/redaction.ts";
|
|
10
10
|
import { sleepSync } from "../utils/sleep.ts";
|
|
@@ -543,9 +543,13 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
543
543
|
} catch {
|
|
544
544
|
/* file does not exist */
|
|
545
545
|
}
|
|
546
|
+
// FIND-10: track whether overflow handling modified the file so we can
|
|
547
|
+
// reuse fileStat for the post-overflow size check (avoids redundant stat).
|
|
548
|
+
let overflowHandled = false;
|
|
546
549
|
if (!isTerminal && fileStat) {
|
|
547
550
|
const stat = fileStat;
|
|
548
551
|
if (stat.size > MAX_EVENTS_BYTES) {
|
|
552
|
+
overflowHandled = true;
|
|
549
553
|
try {
|
|
550
554
|
compactEventLog(eventsPath);
|
|
551
555
|
} catch (error) {
|
|
@@ -564,11 +568,17 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
564
568
|
}
|
|
565
569
|
}
|
|
566
570
|
}
|
|
571
|
+
// FIND-10: collapse redundant stat. If no overflow handling occurred,
|
|
572
|
+
// the file hasn't changed since fileStat — reuse it instead of re-stat'ing.
|
|
567
573
|
let sizeCheckStat: fs.Stats | undefined;
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
574
|
+
if (overflowHandled) {
|
|
575
|
+
try {
|
|
576
|
+
sizeCheckStat = await fs.promises.stat(eventsPath).catch(() => undefined);
|
|
577
|
+
} catch {
|
|
578
|
+
/* file does not exist */
|
|
579
|
+
}
|
|
580
|
+
} else {
|
|
581
|
+
sizeCheckStat = fileStat;
|
|
572
582
|
}
|
|
573
583
|
try {
|
|
574
584
|
if (sizeCheckStat && sizeCheckStat.size > MAX_EVENTS_BYTES) {
|
|
@@ -583,25 +593,44 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
583
593
|
logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
|
|
584
594
|
}
|
|
585
595
|
|
|
596
|
+
// FIND-10: post-append stat captured from the same fd (non-worker path)
|
|
597
|
+
// for reuse in the cache update below, avoiding a redundant path stat.
|
|
598
|
+
let postAppendStat: fs.Stats | undefined;
|
|
586
599
|
if (!skippedDueToSize) {
|
|
587
600
|
const line = JSON.stringify(redactSecrets(fullEvent)) + "\n";
|
|
588
601
|
// Phase 1.5: when worker atomic writer is enabled, append via worker.
|
|
589
602
|
if (isWorkerAtomicWriterEnabled()) {
|
|
590
603
|
await appendFileViaWorker(eventsPath, line);
|
|
604
|
+
// Worker path: fsync via a separate open (worker manages its own fd).
|
|
605
|
+
const fd = await fs.promises.open(eventsPath, "r+");
|
|
606
|
+
try {
|
|
607
|
+
await fd.sync();
|
|
608
|
+
} finally {
|
|
609
|
+
await fd.close();
|
|
610
|
+
}
|
|
591
611
|
} else {
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
612
|
+
// FIND-10: single-fd append+fsync. Opens in append mode, writes,
|
|
613
|
+
// fsyncs on the SAME fd, then closes — eliminating the separate
|
|
614
|
+
// open("r+") + sync that previously doubled the fd count. The
|
|
615
|
+
// fsync (seq-integrity protection) is preserved exactly: it still
|
|
616
|
+
// closes the crash window between append and persistSequence.
|
|
617
|
+
const fd = await fs.promises.open(eventsPath, "a");
|
|
618
|
+
try {
|
|
619
|
+
await fd.appendFile(line, "utf-8");
|
|
620
|
+
await fd.sync();
|
|
621
|
+
// FIND-10 R1 fix: the cache-optimization fd.stat() must NOT sit in the
|
|
622
|
+
// seq-durability critical path. If it threw (rare — fd invalidated),
|
|
623
|
+
// it would skip persistSequence below and reopen the seq-reuse
|
|
624
|
+
// window the fsync just closed. Guard it; fall back to undefined
|
|
625
|
+
// (the later cache-update takes a path stat instead).
|
|
626
|
+
try {
|
|
627
|
+
postAppendStat = await fd.stat();
|
|
628
|
+
} catch {
|
|
629
|
+
postAppendStat = undefined;
|
|
630
|
+
}
|
|
631
|
+
} finally {
|
|
632
|
+
await fd.close();
|
|
633
|
+
}
|
|
605
634
|
}
|
|
606
635
|
// FIX: Persist sequence AFTER successful appendFile to ensure sidecar
|
|
607
636
|
// is only updated when the event is definitively written. If appendFile
|
|
@@ -609,7 +638,11 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
609
638
|
// preventing sequence reuse on restart.
|
|
610
639
|
persistSequence(eventsPath, seq);
|
|
611
640
|
}
|
|
641
|
+
// FIND-10: track whether compaction happened after the append so the
|
|
642
|
+
// cache-update stat can safely reuse postAppendStat (file unchanged).
|
|
643
|
+
let compactedAfterAppend = false;
|
|
612
644
|
if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
|
|
645
|
+
compactedAfterAppend = true;
|
|
613
646
|
try {
|
|
614
647
|
compactEventLog(eventsPath);
|
|
615
648
|
} catch (error) {
|
|
@@ -626,11 +659,18 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
626
659
|
// Only update the cache here (the sidecar persist is already done).
|
|
627
660
|
const finalSeq = fullEvent.metadata?.seq ?? 0;
|
|
628
661
|
try {
|
|
662
|
+
// FIND-10: reuse post-append fd stat when available and no compaction
|
|
663
|
+
// happened after the append (file unchanged). Falls back to path stat
|
|
664
|
+
// for the worker path, skipped events, or post-compaction cases.
|
|
629
665
|
let statResult: fs.Stats | undefined;
|
|
630
|
-
|
|
631
|
-
statResult =
|
|
632
|
-
}
|
|
633
|
-
|
|
666
|
+
if (postAppendStat && !compactedAfterAppend) {
|
|
667
|
+
statResult = postAppendStat;
|
|
668
|
+
} else {
|
|
669
|
+
try {
|
|
670
|
+
statResult = await fs.promises.stat(eventsPath).catch(() => undefined);
|
|
671
|
+
} catch {
|
|
672
|
+
/* file may not exist */
|
|
673
|
+
}
|
|
634
674
|
}
|
|
635
675
|
if (statResult) {
|
|
636
676
|
if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
|
|
@@ -998,6 +1038,7 @@ export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent,
|
|
|
998
1038
|
});
|
|
999
1039
|
}, bufferMs);
|
|
1000
1040
|
bufferedTimers.set(eventsPath, timer);
|
|
1041
|
+
timer.unref();
|
|
1001
1042
|
}
|
|
1002
1043
|
});
|
|
1003
1044
|
}
|
|
@@ -1216,24 +1257,38 @@ export function readEventsCursor(eventsPath: string, options: EventCursorOptions
|
|
|
1216
1257
|
};
|
|
1217
1258
|
}
|
|
1218
1259
|
|
|
1219
|
-
//
|
|
1220
|
-
//
|
|
1221
|
-
//
|
|
1222
|
-
//
|
|
1223
|
-
//
|
|
1260
|
+
// FIND-05 default path: byte-level tail read (last 4MB) instead of
|
|
1261
|
+
// full-file read. Bounds CPU to O(tail bytes) instead of O(total
|
|
1262
|
+
// events). The legacy readEvents() full parse path is preserved for
|
|
1263
|
+
// callers that explicitly need the full history (e.g. tests that
|
|
1264
|
+
// assert exact contents) and as a small-file fallback.
|
|
1265
|
+
//
|
|
1266
|
+
// The 5000-event tail cap and the "event-log.cursor-full-read"
|
|
1267
|
+
// warning are preserved. A separate cursor-tail-truncated warning is
|
|
1268
|
+
// emitted whenever the file exceeds the 4MB tail budget, signalling
|
|
1269
|
+
// that a prefix was dropped and callers should pass fromByteOffset for
|
|
1270
|
+
// streaming reads.
|
|
1271
|
+
const TAIL_BYTES = 4 * 1024 * 1024; // 4 MB
|
|
1272
|
+
const TAIL_EVENT_CAP = 5000;
|
|
1224
1273
|
const sinceSeq = positiveInteger(options.sinceSeq) ?? 0;
|
|
1225
1274
|
const limit = positiveInteger(options.limit);
|
|
1226
|
-
|
|
1227
|
-
const
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1275
|
+
|
|
1276
|
+
const tail = readJsonlTail<TeamEvent>(eventsPath, TAIL_BYTES);
|
|
1277
|
+
let all = tail.items;
|
|
1278
|
+
if (tail.truncated) {
|
|
1279
|
+
logInternalError("event-log.cursor-tail-truncated", {
|
|
1280
|
+
eventsPath,
|
|
1281
|
+
returned: all.length,
|
|
1282
|
+
tailBytes: TAIL_BYTES,
|
|
1283
|
+
});
|
|
1284
|
+
}
|
|
1285
|
+
if (all.length > TAIL_EVENT_CAP) {
|
|
1231
1286
|
logInternalError(
|
|
1232
1287
|
"event-log.cursor-full-read",
|
|
1233
|
-
new Error(`readEventsCursor read
|
|
1288
|
+
new Error(`readEventsCursor tail read dropped events from a larger log; pass fromByteOffset for incremental reads`),
|
|
1234
1289
|
`eventsPath=${eventsPath}`,
|
|
1235
1290
|
);
|
|
1236
|
-
all = all.slice(-
|
|
1291
|
+
all = all.slice(-TAIL_EVENT_CAP);
|
|
1237
1292
|
}
|
|
1238
1293
|
const filtered = all.filter((event) => (event.metadata?.seq ?? 0) > sinceSeq);
|
|
1239
1294
|
const events = limit !== undefined ? filtered.slice(0, limit) : filtered;
|