pi-crew 0.9.9 → 0.9.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +330 -0
- package/docs/fixes/v0.9.10/locks-fix-verify.md +3 -0
- package/docs/fixes/v0.9.10/smoke-test.md +12 -0
- package/package.json +1 -1
- package/src/config/role-tools.ts +39 -6
- package/src/extension/team-tool/doctor.ts +41 -18
- package/src/runtime/async-runner.ts +70 -74
- package/src/runtime/background-runner.ts +13 -2
- package/src/runtime/child-pi.ts +122 -22
- package/src/runtime/compact-pipeline.ts +56 -0
- package/src/runtime/compact-stages/ansi-strip-stage.ts +25 -0
- package/src/runtime/compact-stages/blank-collapse-stage.ts +31 -0
- package/src/runtime/compact-stages/deduplicate-stage.ts +34 -0
- package/src/runtime/compact-stages/head-snap-stage.ts +57 -0
- package/src/runtime/compact-stages/index.ts +13 -0
- package/src/runtime/compact-stages/tail-capture-stage.ts +72 -0
- package/src/runtime/compact-stages/truncation-stage.ts +71 -0
- package/src/runtime/handoff-manager.ts +10 -0
- package/src/runtime/important-line-classifier.ts +130 -0
- package/src/runtime/iteration-hooks.ts +7 -19
- package/src/runtime/live-session-runtime.ts +50 -1
- package/src/runtime/model-fallback.ts +29 -1
- package/src/runtime/role-permission.ts +5 -21
- package/src/runtime/stream-preview.ts +9 -2
- package/src/runtime/task-output-context.ts +161 -27
- package/src/runtime/task-runner/prompt-builder.ts +1 -0
- package/src/runtime/task-runner.ts +76 -15
- package/src/state/artifact-store.ts +22 -2
- package/src/state/locks.ts +16 -0
- package/src/state/state-store.ts +8 -2
- package/src/ui/live-run-sidebar.ts +6 -1
- package/src/ui/loaders.ts +24 -4
- package/src/ui/run-dashboard.ts +6 -1
- package/src/ui/run-event-bus.ts +1 -1
- package/src/ui/run-snapshot-cache.ts +50 -16
- package/src/ui/widget/index.ts +27 -5
- package/src/ui/widget/widget-renderer.ts +43 -13
- package/src/utils/redaction.ts +66 -32
- package/src/utils/visual.ts +6 -0
- package/src/ui/crew-widget.ts +0 -544
|
@@ -5,6 +5,8 @@ import { writeArtifact } from "../state/artifact-store.ts";
|
|
|
5
5
|
import { resolveRealContainedPath } from "../utils/safe-paths.ts";
|
|
6
6
|
import type { WorkflowStep } from "../workflows/workflow-config.ts";
|
|
7
7
|
import { pruneToolOutputs, type ToolResultEntry, type FileEditEvent, DEFAULT_PRUNE_CONFIG } from "./tool-output-pruner.ts";
|
|
8
|
+
import { applyCompactPipeline } from "./compact-pipeline.ts";
|
|
9
|
+
import { ANSI_STRIP_STAGE, BLANK_COLLAPSE_STAGE, TruncationStage } from "./compact-stages/index.ts";
|
|
8
10
|
|
|
9
11
|
export interface DependencyContextEntry {
|
|
10
12
|
taskId: string;
|
|
@@ -19,7 +21,14 @@ export interface DependencyContextEntry {
|
|
|
19
21
|
|
|
20
22
|
export interface DependencyOutputContext {
|
|
21
23
|
dependencies: DependencyContextEntry[];
|
|
22
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Each shared artifact read, truncated for inline injection. When truncation
|
|
26
|
+
* is materially lossy (file size > 2× MAX_RESULT_INLINE_BYTES) the FULL
|
|
27
|
+
* content is also teed to `${artifactsRoot}/tee/${taskId}-${name}.full.txt`
|
|
28
|
+
* and the path is exposed via `fullOutputPath` so the downstream worker
|
|
29
|
+
* can `read` it back if it needs the dropped middle.
|
|
30
|
+
*/
|
|
31
|
+
sharedReads: Array<{ name: string; path: string; content: string; fullOutputPath?: string }>;
|
|
23
32
|
}
|
|
24
33
|
|
|
25
34
|
function containedExists(filePath: string, baseDir?: string): boolean {
|
|
@@ -39,35 +48,127 @@ function containedExists(filePath: string, baseDir?: string): boolean {
|
|
|
39
48
|
* (24K/40K/80K) which truncated the same artifact differently depending on
|
|
40
49
|
* which code path read it.
|
|
41
50
|
*/
|
|
42
|
-
const MAX_RESULT_INLINE_BYTES = 32_000;
|
|
51
|
+
export const MAX_RESULT_INLINE_BYTES = 32_000;
|
|
43
52
|
|
|
44
|
-
|
|
45
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Read a file and return its content, truncating to a head+tail slice if it
|
|
55
|
+
* exceeds {@link MAX_RESULT_INLINE_BYTES} characters. Multi-byte UTF-8
|
|
56
|
+
* sequences are preserved by reading the full file as a UTF-8 string and
|
|
57
|
+
* slicing by character count (not raw bytes).
|
|
58
|
+
*/
|
|
59
|
+
export interface TeeRecoveryOptions {
|
|
60
|
+
/** Absolute path to write the full (non-truncated) content to. */
|
|
61
|
+
fullOutputPath: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface ReadIfSmallTeeResult {
|
|
65
|
+
/** Truncated content (or full content when no truncation). */
|
|
66
|
+
content: string;
|
|
67
|
+
/** Set only when tee was actually written (file size > 2× threshold + write succeeded). */
|
|
68
|
+
fullOutputPath?: string;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Sanitize a taskId / artifactName into a flat tee filename. Any character
|
|
73
|
+
* outside [A-Za-z0-9._-] is replaced with underscore so the resulting path
|
|
74
|
+
* is always single-segment and cannot escape the tee directory.
|
|
75
|
+
*/
|
|
76
|
+
function safeTeeName(taskId: string, artifactName: string): string {
|
|
77
|
+
const safe = (s: string): string => s.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
78
|
+
return `${safe(taskId)}-${safe(artifactName)}.full.txt`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Canonical tee path for a shared artifact read.
|
|
83
|
+
*
|
|
84
|
+
* Format: `${artifactsRoot}/tee/${taskId}-${artifactName}.full.txt`
|
|
85
|
+
*
|
|
86
|
+
* The downstream worker prompt includes this path so the worker can `read`
|
|
87
|
+
* the full content when it needs the dropped middle.
|
|
88
|
+
*/
|
|
89
|
+
export function teePathForArtifact(artifactsRoot: string, taskId: string, artifactName: string): string {
|
|
90
|
+
return path.join(artifactsRoot, "tee", safeTeeName(taskId, artifactName));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Best-effort tee write. Returns true on success, false on any error (write
|
|
95
|
+
* failures are silent — tee is enhancement, never a hard dependency). The
|
|
96
|
+
* truncated inline content is still returned by the caller either way.
|
|
97
|
+
*/
|
|
98
|
+
function writeTeeFile(fullOutputPath: string, content: string): boolean {
|
|
46
99
|
try {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
100
|
+
fs.mkdirSync(path.dirname(fullOutputPath), { recursive: true });
|
|
101
|
+
fs.writeFileSync(fullOutputPath, content, "utf-8");
|
|
102
|
+
return true;
|
|
103
|
+
} catch {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Read a file with optional tee-recovery (P1-A). Returns the truncated
|
|
110
|
+
* content AND (when tee was actually written) the absolute path to the full
|
|
111
|
+
* file. Returns undefined if the file cannot be read at all.
|
|
112
|
+
*
|
|
113
|
+
* Tee threshold: only when content.length > 2 * MAX_RESULT_INLINE_BYTES
|
|
114
|
+
* (the head+tail is materially lossy — small over-threshold files are not
|
|
115
|
+
* teed because the inline content is mostly intact and the worker can live
|
|
116
|
+
* with the 75/25 split). File content is read once and reused for both the
|
|
117
|
+
* pipeline (truncation) and the tee write (full file).
|
|
118
|
+
*
|
|
119
|
+
* Truncation behavior is unchanged from the P0-A pipeline: ANSI strip +
|
|
120
|
+
* blank collapse BEFORE truncation, important-line preservation (P0-B)
|
|
121
|
+
* inside TruncationStage, marker wording matches the pre-P1-A `readIfSmall`
|
|
122
|
+
* output exactly (L4 backward-compat).
|
|
123
|
+
*/
|
|
124
|
+
export function readIfSmallWithTee(
|
|
125
|
+
filePath: string,
|
|
126
|
+
opts: { baseDir?: string; tee?: TeeRecoveryOptions } = {},
|
|
127
|
+
): ReadIfSmallTeeResult | undefined {
|
|
128
|
+
const maxChars = MAX_RESULT_INLINE_BYTES;
|
|
129
|
+
try {
|
|
130
|
+
const safePath = opts.baseDir ? resolveRealContainedPath(opts.baseDir, filePath) : filePath;
|
|
131
|
+
const content = fs.readFileSync(safePath, "utf-8");
|
|
132
|
+
if (content.length > maxChars) {
|
|
133
|
+
let fullOutputPath: string | undefined;
|
|
134
|
+
// Tee only when truncation is materially lossy (>2× threshold).
|
|
135
|
+
if (opts.tee && content.length > maxChars * 2) {
|
|
136
|
+
if (writeTeeFile(opts.tee.fullOutputPath, content)) {
|
|
137
|
+
fullOutputPath = opts.tee.fullOutputPath;
|
|
138
|
+
}
|
|
62
139
|
}
|
|
63
|
-
|
|
140
|
+
const result = applyCompactPipeline(content, [
|
|
141
|
+
ANSI_STRIP_STAGE,
|
|
142
|
+
BLANK_COLLAPSE_STAGE,
|
|
143
|
+
new TruncationStage(maxChars, {
|
|
144
|
+
preserveImportant: true,
|
|
145
|
+
marker: { verb: "truncated", unit: "chars", headSeparator: "\n\n", tailSeparator: "\n" },
|
|
146
|
+
}),
|
|
147
|
+
]);
|
|
148
|
+
return fullOutputPath ? { content: result.text, fullOutputPath } : { content: result.text };
|
|
64
149
|
}
|
|
65
|
-
return
|
|
150
|
+
return { content };
|
|
66
151
|
} catch {
|
|
67
152
|
return undefined;
|
|
68
153
|
}
|
|
69
154
|
}
|
|
70
155
|
|
|
156
|
+
/**
|
|
157
|
+
* Read a file and return its content, truncating to a head+tail slice if it
|
|
158
|
+
* exceeds {@link MAX_RESULT_INLINE_BYTES} characters. Multi-byte UTF-8
|
|
159
|
+
* sequences are preserved by reading the full file as a UTF-8 string and
|
|
160
|
+
* slicing by character count (not raw bytes).
|
|
161
|
+
*
|
|
162
|
+
* Thin wrapper around {@link readIfSmallWithTee} for backward compatibility
|
|
163
|
+
* — callers that do not need tee-recovery metadata get just the content
|
|
164
|
+
* string. New tee-recovery call sites should use {@link readIfSmallWithTee}
|
|
165
|
+
* directly so they can include the full output path in the worker prompt.
|
|
166
|
+
*/
|
|
167
|
+
export function readIfSmall(filePath: string, baseDir?: string): string | undefined {
|
|
168
|
+
const result = readIfSmallWithTee(filePath, { baseDir });
|
|
169
|
+
return result?.content;
|
|
170
|
+
}
|
|
171
|
+
|
|
71
172
|
function safeSharedName(name: string): string {
|
|
72
173
|
const normalized = name.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
73
174
|
if (!normalized || normalized.split("/").some((segment) => segment === "..") || path.isAbsolute(normalized)) throw new Error(`Invalid shared artifact name: ${name}`);
|
|
@@ -127,6 +228,7 @@ function aggregateUsage(task: TeamTaskState): DependencyContextEntry["usage"] {
|
|
|
127
228
|
function pruneSharedReads(
|
|
128
229
|
reads: Array<{ name: string; path: string; content: string }>,
|
|
129
230
|
dependencies: DependencyContextEntry[],
|
|
231
|
+
artifactsRoot: string,
|
|
130
232
|
): Array<{ name: string; path: string; content: string }> {
|
|
131
233
|
if (reads.length === 0) return reads;
|
|
132
234
|
// Convert shared reads to tool result entries (ordered oldest → newest
|
|
@@ -140,15 +242,19 @@ function pruneSharedReads(
|
|
|
140
242
|
// Collect file edit events from dependency artifacts produced to shared/.
|
|
141
243
|
// A dependency that wrote a shared file after an earlier read invalidates
|
|
142
244
|
// that read (the content is now stale relative to the latest version).
|
|
143
|
-
|
|
245
|
+
// Artifact entries from listTaskArtifacts() are already relative to
|
|
246
|
+
// artifactsRoot (e.g. "shared/foo.md"), so resolve directly against
|
|
247
|
+
// artifactsRoot — NOT against a "shared" subdirectory (which would
|
|
248
|
+
// double-prefix to <artifactsRoot>/shared/shared/foo.md).
|
|
144
249
|
const fileEdits: FileEditEvent[] = [];
|
|
145
250
|
for (let depIndex = 0; depIndex < dependencies.length; depIndex++) {
|
|
146
251
|
const dep = dependencies[depIndex]!;
|
|
147
252
|
const produced = dep.artifactsProduced ?? [];
|
|
148
253
|
for (const artifact of produced) {
|
|
149
254
|
if (typeof artifact !== "string") continue;
|
|
150
|
-
// Map artifact path
|
|
151
|
-
|
|
255
|
+
// Map artifact path (relative to artifactsRoot) to absolute and
|
|
256
|
+
// check against read targets.
|
|
257
|
+
fileEdits.push({ target: path.resolve(artifactsRoot, artifact), index: reads.length + depIndex });
|
|
152
258
|
}
|
|
153
259
|
}
|
|
154
260
|
const pruned = pruneToolOutputs(entries, DEFAULT_PRUNE_CONFIG);
|
|
@@ -175,13 +281,33 @@ export function collectDependencyOutputContext(manifest: TeamRunManifest, tasks:
|
|
|
175
281
|
});
|
|
176
282
|
const rawSharedReads = (step.reads === false ? [] : step.reads ?? []).map((name) => {
|
|
177
283
|
const filePath = sharedPath(manifest, name);
|
|
178
|
-
|
|
284
|
+
// P1-A tee-recovery: when the shared artifact is large enough that the
|
|
285
|
+
// 75/25 head+tail split is materially lossy (>2× MAX_RESULT_INLINE_BYTES),
|
|
286
|
+
// tee the full content to ${artifactsRoot}/tee/${taskId}-${name}.full.txt
|
|
287
|
+
// and expose the path so the downstream worker can `read` the full file
|
|
288
|
+
// if it needs the dropped middle. The truncated content is still
|
|
289
|
+
// included inline; tee is an enhancement, not a hard dependency. Tee
|
|
290
|
+
// write is best-effort (writeTeeFile swallows I/O errors and the result
|
|
291
|
+
// simply omits fullOutputPath in that case).
|
|
292
|
+
const teePath = teePathForArtifact(manifest.artifactsRoot, task.id, name);
|
|
293
|
+
const teeResult = readIfSmallWithTee(filePath, {
|
|
294
|
+
baseDir: path.resolve(manifest.artifactsRoot, "shared"),
|
|
295
|
+
tee: { fullOutputPath: teePath },
|
|
296
|
+
});
|
|
297
|
+
if (teeResult === undefined) return { name, path: filePath, content: "" };
|
|
298
|
+
const entry: { name: string; path: string; content: string; fullOutputPath?: string } = {
|
|
299
|
+
name,
|
|
300
|
+
path: filePath,
|
|
301
|
+
content: teeResult.content,
|
|
302
|
+
};
|
|
303
|
+
if (teeResult.fullOutputPath) entry.fullOutputPath = teeResult.fullOutputPath;
|
|
304
|
+
return entry;
|
|
179
305
|
}).filter((item) => item.content.trim().length > 0);
|
|
180
306
|
// Apply staleness-aware pruning to shared reads: drops superseded reads
|
|
181
307
|
// (same file re-read with different selectors) and replaces stale large
|
|
182
308
|
// outputs with compact digest notices before injecting into the worker
|
|
183
309
|
// prompt. OPT-IN: default config protects recent results.
|
|
184
|
-
const sharedReads = pruneSharedReads(rawSharedReads, dependencies);
|
|
310
|
+
const sharedReads = pruneSharedReads(rawSharedReads, dependencies, manifest.artifactsRoot);
|
|
185
311
|
return { dependencies, sharedReads };
|
|
186
312
|
}
|
|
187
313
|
|
|
@@ -198,7 +324,15 @@ export function renderDependencyOutputContext(context: DependencyOutputContext):
|
|
|
198
324
|
}
|
|
199
325
|
if (context.sharedReads.length) {
|
|
200
326
|
parts.push("# Shared Run Context Reads", "");
|
|
201
|
-
for (const read of context.sharedReads)
|
|
327
|
+
for (const read of context.sharedReads) {
|
|
328
|
+
parts.push(`## shared/${read.name}`, `Path: ${read.path}`);
|
|
329
|
+
// P1-A tee-recovery hint: when the file was materially truncated
|
|
330
|
+
// (>2× threshold) the full content was teed to fullOutputPath so the
|
|
331
|
+
// worker can read the dropped middle if needed. The path is inside
|
|
332
|
+
// artifactsRoot/tee/ and goes through the normal permission gate.
|
|
333
|
+
if (read.fullOutputPath) parts.push(`Full output (if you need the missing middle): ${read.fullOutputPath}`);
|
|
334
|
+
parts.push("", read.content.trim(), "");
|
|
335
|
+
}
|
|
202
336
|
}
|
|
203
337
|
return parts.join("\n").trim();
|
|
204
338
|
}
|
|
@@ -30,6 +30,7 @@ function readOnlyRoleInstructions(role: string): string {
|
|
|
30
30
|
"- Do not use shell redirects, heredocs, in-place edits, package installs, git commit/merge/rebase/reset/checkout, or other state-mutating commands.",
|
|
31
31
|
"- If implementation changes are needed, report exact recommendations instead of applying them.",
|
|
32
32
|
"- Prefer read/grep/find/listing tools and read-only git inspection commands.",
|
|
33
|
+
"- Your final RESULT TEXT is persisted automatically by the runner (as a result artifact and, if the step declares `output:`, to a shared file). To deliver a plan, report, or findings, EMIT THEM AS TEXT in your final result — do NOT try to write a file yourself.",
|
|
33
34
|
].join("\n");
|
|
34
35
|
}
|
|
35
36
|
|
|
@@ -763,7 +763,30 @@ export async function runTeamTask(
|
|
|
763
763
|
"",
|
|
764
764
|
);
|
|
765
765
|
if (!error) break;
|
|
766
|
-
|
|
766
|
+
let nextModel = attemptModels[i + 1];
|
|
767
|
+
// FIX 1 (task packet 01_01-agent): when the precomputed attempt
|
|
768
|
+
// chain is exhausted but the failure is retryable, do a one-shot
|
|
769
|
+
// re-resolve via buildConfiguredModelRouting with the failed
|
|
770
|
+
// model as parent. This finds alternative providers/models the
|
|
771
|
+
// original chain missed (e.g. a registry gained new fallbacks
|
|
772
|
+
// after the precompute, or the precompute ran before the parent
|
|
773
|
+
// model was known). If a different candidate is found, use it as
|
|
774
|
+
// nextModel; otherwise fall through to the existing break.
|
|
775
|
+
if (!nextModel && isRetryableModelFailure(error)) {
|
|
776
|
+
const reResolved = buildConfiguredModelRouting({
|
|
777
|
+
overrideModel: undefined,
|
|
778
|
+
stepModel: undefined,
|
|
779
|
+
teamRoleModel: undefined,
|
|
780
|
+
agentModel: undefined,
|
|
781
|
+
fallbackModels: undefined,
|
|
782
|
+
parentModel: attempt.model,
|
|
783
|
+
modelRegistry: input.modelRegistry,
|
|
784
|
+
cwd: task.cwd,
|
|
785
|
+
scopeModelsPatterns: await resolveTaskScopeModelsPatterns(task.cwd),
|
|
786
|
+
});
|
|
787
|
+
const alt = reResolved.candidates.find((c) => c !== attempt.model);
|
|
788
|
+
if (alt) nextModel = alt;
|
|
789
|
+
}
|
|
767
790
|
if (!nextModel || !isRetryableModelFailure(error)) break;
|
|
768
791
|
logs.push(formatModelAttemptNote(attempt, nextModel), "");
|
|
769
792
|
}
|
|
@@ -1368,19 +1391,57 @@ async function resolveTaskScopeModelsPatterns(cwd: string): Promise<string[]> {
|
|
|
1368
1391
|
* or when there are no retryable error messages.
|
|
1369
1392
|
*/
|
|
1370
1393
|
export function detectRetryableModelFailureFromOutput(parsed: ParsedPiJsonOutput): string | undefined {
|
|
1394
|
+
// Primary signal: pre-extracted `errorMessages` (from pi-json-output parser).
|
|
1395
|
+
// The parser already filters to non-empty trimmed strings from message_end
|
|
1396
|
+
// events.
|
|
1371
1397
|
const messages = parsed.errorMessages;
|
|
1372
|
-
if (
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1398
|
+
if (messages && messages.length > 0) {
|
|
1399
|
+
// Find the first retryable model-failure message
|
|
1400
|
+
// (429 / rate-limit / overloaded / 5xx / ...).
|
|
1401
|
+
const retryable = messages.find((m) => isRetryableModelFailure(m));
|
|
1402
|
+
if (retryable) {
|
|
1403
|
+
// Did the run actually produce real output despite the transient errors?
|
|
1404
|
+
// If finalText / textEvents / patches exist, the model recovered and we
|
|
1405
|
+
// should NOT mark the run as failed — only flag it when the worker
|
|
1406
|
+
// yielded nothing (the 429-only case from the bug report).
|
|
1407
|
+
const hasRealOutput =
|
|
1408
|
+
(parsed.finalText?.trim().length ?? 0) > 0 ||
|
|
1409
|
+
parsed.textEvents.some((t) => t.trim().length > 0) ||
|
|
1410
|
+
(parsed.patches?.length ?? 0) > 0;
|
|
1411
|
+
if (hasRealOutput) return undefined;
|
|
1412
|
+
return `Model returned only retryable errors and no output: ${retryable}`;
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
// Secondary signal (FIX 3, task packet 01_01-agent): inspect a raw
|
|
1416
|
+
// `messageEndEvents` (or `transcript`) array on the parsed output. The
|
|
1417
|
+
// ParsedPiJsonOutput type does not currently declare this field, so we
|
|
1418
|
+
// read it through a local extension cast. Callers that pass it (tests, a
|
|
1419
|
+
// future parser that captures the full event stream) get a second chance
|
|
1420
|
+
// to surface retryable failures. Primary path still wins when it matches.
|
|
1421
|
+
const raw = parsed as ParsedPiJsonOutput & {
|
|
1422
|
+
messageEndEvents?: unknown;
|
|
1423
|
+
transcript?: unknown;
|
|
1424
|
+
};
|
|
1425
|
+
const eventSource = Array.isArray(raw.messageEndEvents)
|
|
1426
|
+
? raw.messageEndEvents
|
|
1427
|
+
: Array.isArray(raw.transcript)
|
|
1428
|
+
? raw.transcript
|
|
1429
|
+
: undefined;
|
|
1430
|
+
if (!eventSource || eventSource.length === 0) return undefined;
|
|
1431
|
+
for (const candidate of eventSource) {
|
|
1432
|
+
if (!candidate || typeof candidate !== "object") continue;
|
|
1433
|
+
const event = candidate as { stopReason?: unknown; errorMessage?: unknown };
|
|
1434
|
+
if (event.stopReason !== "error") continue;
|
|
1435
|
+
if (typeof event.errorMessage !== "string" || event.errorMessage.length === 0) continue;
|
|
1436
|
+
if (!isRetryableModelFailure(event.errorMessage)) continue;
|
|
1437
|
+
// Same real-output gate as the primary signal — don't flag runs that
|
|
1438
|
+
// recovered with real final text / patches.
|
|
1439
|
+
const hasRealOutput =
|
|
1440
|
+
(parsed.finalText?.trim().length ?? 0) > 0 ||
|
|
1441
|
+
parsed.textEvents.some((t) => t.trim().length > 0) ||
|
|
1442
|
+
(parsed.patches?.length ?? 0) > 0;
|
|
1443
|
+
if (hasRealOutput) return undefined;
|
|
1444
|
+
return `Model returned only retryable errors and no output: ${event.errorMessage}`;
|
|
1445
|
+
}
|
|
1446
|
+
return undefined;
|
|
1386
1447
|
}
|
|
@@ -4,7 +4,7 @@ import { createHash } from "node:crypto";
|
|
|
4
4
|
import type { ArtifactDescriptor } from "./types.ts";
|
|
5
5
|
import { atomicWriteFile } from "./atomic-write.ts";
|
|
6
6
|
import { resolveRealContainedPath } from "../utils/safe-paths.ts";
|
|
7
|
-
import { redactSecretString } from "../utils/redaction.ts";
|
|
7
|
+
import { redactSecretString, redactSecrets } from "../utils/redaction.ts";
|
|
8
8
|
|
|
9
9
|
function hashContent(content: string): string {
|
|
10
10
|
return createHash("sha256").update(content).digest("hex");
|
|
@@ -127,7 +127,27 @@ export function writeArtifact(artifactsRoot: string, options: ArtifactWriteOptio
|
|
|
127
127
|
const filePath = resolveInside(artifactsRoot, options.relativePath);
|
|
128
128
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
129
129
|
resolveRealContainedPath(artifactsRoot, path.dirname(filePath));
|
|
130
|
-
|
|
130
|
+
let content = options.content;
|
|
131
|
+
// Structural JSON redaction first — catches quoted-JSON secrets
|
|
132
|
+
// ("api_key":"sk-...") and nested keys that flat redactSecretString misses.
|
|
133
|
+
// The flat scan below still catches free-text patterns (Bearer/JWT/Auth
|
|
134
|
+
// headers) that may live inside JSON string values. See security review M2.
|
|
135
|
+
//
|
|
136
|
+
// Formatting preservation: re-stringify with the SAME indentation as the
|
|
137
|
+
// input so pretty-printed artifacts (e.g. group-join metadata expected by
|
|
138
|
+
// test/integration/phase4-runtime.test.ts to contain `"partial": false`)
|
|
139
|
+
// keep their whitespace. Detect pretty-vs-compact from the raw input.
|
|
140
|
+
const trimmed = content.trim();
|
|
141
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
142
|
+
try {
|
|
143
|
+
const parsed: unknown = JSON.parse(content);
|
|
144
|
+
const isPretty = /\n|"\s*:\s/.test(content);
|
|
145
|
+
content = JSON.stringify(redactSecrets(parsed), null, isPretty ? 2 : undefined);
|
|
146
|
+
} catch {
|
|
147
|
+
// not valid JSON — fall through to flat redaction only
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
content = redactSecretString(content);
|
|
131
151
|
atomicWriteFile(filePath, content);
|
|
132
152
|
// Compute hash on written bytes for integrity verification.
|
|
133
153
|
// Read back the actual file content to handle atomicWrite fallback path
|
package/src/state/locks.ts
CHANGED
|
@@ -292,6 +292,17 @@ export function withFileLockSync<T>(filePath: string, fn: () => T, options: RunL
|
|
|
292
292
|
// append, or even the lock acquisition itself) would race with the lock.
|
|
293
293
|
const lockFile = `${filePath}.lock`;
|
|
294
294
|
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
|
295
|
+
// FIX (Round 29): re-entrance guard — mirrors withRunLockSync below.
|
|
296
|
+
// When the same call stack already holds the file lock (e.g.
|
|
297
|
+
// registerWorker -> cleanupOrphanWorkers -> readRegistry), the second
|
|
298
|
+
// acquisition would otherwise read its own freshly-written lock file
|
|
299
|
+
// (same pid, fresh createdAt), fail the steal check, and deadlock for
|
|
300
|
+
// the full staleMs window. Strace-confirmed in
|
|
301
|
+
// .github/issues/pre-existing-2026-06-10/04-orphan-worker-registry-tests.md:75-86.
|
|
302
|
+
const existingToken = fileLockHeldByUs.get(lockFile);
|
|
303
|
+
if (existingToken) {
|
|
304
|
+
return fn();
|
|
305
|
+
}
|
|
295
306
|
// FIX: Validate the parent directory is not a symlink BEFORE calling mkdirSync.
|
|
296
307
|
// Between mkdir and lock acquisition, an attacker could plant a symlink.
|
|
297
308
|
if (!isSymlinkSafePath(path.dirname(lockFile))) throw new Error("Refusing: parent of lock directory is a symlink");
|
|
@@ -322,10 +333,12 @@ export function withFileLockSync<T>(filePath: string, fn: () => T, options: RunL
|
|
|
322
333
|
}
|
|
323
334
|
}
|
|
324
335
|
if (token === "") throw new Error(`Run '${path.basename(lockFile)}' is locked by another operation.`);
|
|
336
|
+
fileLockHeldByUs.set(lockFile, token);
|
|
325
337
|
try {
|
|
326
338
|
return fn();
|
|
327
339
|
} finally {
|
|
328
340
|
// Token-guarded release: don't rm the lock if it has been stolen.
|
|
341
|
+
fileLockHeldByUs.delete(lockFile);
|
|
329
342
|
releaseLock(lockFile, token);
|
|
330
343
|
}
|
|
331
344
|
}
|
|
@@ -353,6 +366,9 @@ export function withRunLockSync<T>(manifest: TeamRunManifest, fn: () => T, optio
|
|
|
353
366
|
// already held by this call stack (handleResume -> executeTeamRun ->
|
|
354
367
|
// executeTeamRunCore), we skip re-acquisition to avoid deadlock.
|
|
355
368
|
const runLockHeldByUs = new Map<string, string>(); // filePath -> token
|
|
369
|
+
// Round 29: parallel map for withFileLockSync re-entrance. See the comment
|
|
370
|
+
// at the top of withFileLockSync for the full deadlock mechanism.
|
|
371
|
+
const fileLockHeldByUs = new Map<string, string>(); // lockFile -> token
|
|
356
372
|
|
|
357
373
|
export async function withRunLock<T>(manifest: TeamRunManifest, fn: () => Promise<T>, options: RunLockOptions = {}): Promise<T> {
|
|
358
374
|
const filePath = lockPath(manifest);
|
package/src/state/state-store.ts
CHANGED
|
@@ -634,7 +634,10 @@ export function loadRunManifestById(cwd: string, runId: string): { manifest: Tea
|
|
|
634
634
|
// between the final stat and the read. Callers needing strict consistency
|
|
635
635
|
// MUST use withRunLock() around load+modify+save.
|
|
636
636
|
if (attempts > 0) {
|
|
637
|
-
|
|
637
|
+
// Round 19: downgrade to debug — retry-loop instability is expected under
|
|
638
|
+
// concurrent writes (live team runs constantly append to tasks.json).
|
|
639
|
+
// This is best-effort by design; strict consistency requires withRunLock().
|
|
640
|
+
console.debug(`[state-store] loadRunManifestById: retry loop detected instability for run ${runId} after ${attempts} attempt(s) — best-effort only, use withRunLock() for strict consistency`);
|
|
638
641
|
}
|
|
639
642
|
// NOTE: manifest mtime may legitimately be >= tasks mtime because
|
|
640
643
|
// saveManifestAndTasksAtomicSync writes manifest before tasks. However,
|
|
@@ -724,7 +727,10 @@ export async function loadRunManifestByIdAsync(cwd: string, runId: string): Prom
|
|
|
724
727
|
// between the final stat and the read. Callers needing strict consistency
|
|
725
728
|
// MUST use withRunLock() around load+modify+save.
|
|
726
729
|
if (attempts > 0) {
|
|
727
|
-
|
|
730
|
+
// Round 19: downgrade to debug — retry-loop instability is expected under
|
|
731
|
+
// concurrent writes (live team runs constantly append to tasks.json).
|
|
732
|
+
// This is best-effort by design; strict consistency requires withRunLock().
|
|
733
|
+
console.debug(`[state-store] loadRunManifestByIdAsync: retry loop detected instability for run ${runId} after ${attempts} attempt(s) — best-effort only, use withRunLock() for strict consistency`);
|
|
728
734
|
}
|
|
729
735
|
// NOTE: manifest mtime may legitimately be >= tasks mtime because
|
|
730
736
|
// saveManifestAndTasksAtomicSync writes manifest before tasks. However,
|
|
@@ -76,7 +76,12 @@ export class LiveRunSidebar {
|
|
|
76
76
|
this.config = input.config ?? {};
|
|
77
77
|
this.snapshotCache = input.snapshotCache;
|
|
78
78
|
this.unsubscribeTheme = subscribeThemeChange(input.theme, () => this.invalidate());
|
|
79
|
-
this.unsubscribeEventBus =
|
|
79
|
+
this.unsubscribeEventBus = (() => {
|
|
80
|
+
const unsub1 = runEventBus.onChannel("run:state", () => this.invalidate());
|
|
81
|
+
const unsub2 = runEventBus.onChannel("worker:lifecycle", () => this.invalidate());
|
|
82
|
+
const unsub3 = runEventBus.onChannel("ui:invalidate", () => this.invalidate());
|
|
83
|
+
return () => { unsub1(); unsub2(); unsub3(); };
|
|
84
|
+
})();
|
|
80
85
|
}
|
|
81
86
|
|
|
82
87
|
private buildSignature(manifestStatus: string, tasks: TeamTaskState[], agents: ReturnType<typeof readCrewAgents>, waitingCount: number, snapshot?: RunUiSnapshot): string {
|
package/src/ui/loaders.ts
CHANGED
|
@@ -113,23 +113,43 @@ export class CountdownTimer {
|
|
|
113
113
|
private readonly timeoutMs: number;
|
|
114
114
|
private timer: ReturnType<typeof setTimeout> | undefined;
|
|
115
115
|
private expired = false;
|
|
116
|
+
private lastEmittedSeconds = -1;
|
|
116
117
|
|
|
117
118
|
constructor(options: CountdownTimerOptions) {
|
|
118
119
|
this.timeoutMs = Math.max(0, options.timeoutMs);
|
|
119
120
|
this.onTick = options.onTick;
|
|
120
121
|
this.onExpire = options.onExpire;
|
|
121
122
|
this.startedAt = Date.now();
|
|
122
|
-
this.
|
|
123
|
+
this.lastEmittedSeconds = this.secondsLeft();
|
|
124
|
+
this.onTick(this.lastEmittedSeconds);
|
|
123
125
|
if (this.timeoutMs === 0) {
|
|
124
126
|
this.emitExpire();
|
|
125
127
|
return;
|
|
126
128
|
}
|
|
127
|
-
this.
|
|
129
|
+
this.scheduleNextTick();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Schedule the next tick via recursive setTimeout. Each tick re-emits the
|
|
134
|
+
* current `secondsLeft()` only if it differs from the last emitted value
|
|
135
|
+
* (lastEmittedSeconds guard). This makes the countdown correct even under
|
|
136
|
+
* event-loop pressure: if the previous tick fired 1.2s late, the next
|
|
137
|
+
* tick still emits the right value for the current second rather than
|
|
138
|
+
* skipping it (the pre-fix `setInterval` could SKIP a second value when
|
|
139
|
+
* the loop was busy, producing [3,2,0] instead of [3,2,1,0] in tests).
|
|
140
|
+
*/
|
|
141
|
+
private scheduleNextTick(): void {
|
|
142
|
+
this.timer = setTimeout(() => {
|
|
128
143
|
const seconds = this.secondsLeft();
|
|
129
|
-
this.
|
|
144
|
+
if (seconds !== this.lastEmittedSeconds) {
|
|
145
|
+
this.lastEmittedSeconds = seconds;
|
|
146
|
+
this.onTick(seconds);
|
|
147
|
+
}
|
|
130
148
|
if (seconds <= 0) {
|
|
131
149
|
this.emitExpire();
|
|
150
|
+
return;
|
|
132
151
|
}
|
|
152
|
+
this.scheduleNextTick();
|
|
133
153
|
}, 1000);
|
|
134
154
|
// Defense-in-depth: never let the countdown timer keep the event loop
|
|
135
155
|
// alive. If dispose() is missed (e.g. UI unmount race), the timer must
|
|
@@ -151,7 +171,7 @@ export class CountdownTimer {
|
|
|
151
171
|
|
|
152
172
|
dispose(): void {
|
|
153
173
|
if (this.timer === undefined) return;
|
|
154
|
-
|
|
174
|
+
clearTimeout(this.timer);
|
|
155
175
|
this.timer = undefined;
|
|
156
176
|
}
|
|
157
177
|
}
|
package/src/ui/run-dashboard.ts
CHANGED
|
@@ -294,7 +294,12 @@ export class RunDashboard implements DashboardComponent {
|
|
|
294
294
|
this.theme = asCrewTheme(theme);
|
|
295
295
|
this.options = options;
|
|
296
296
|
this.unsubscribeTheme = subscribeThemeChange(theme, () => this.invalidateAndRender());
|
|
297
|
-
this.unsubscribeEventBus =
|
|
297
|
+
this.unsubscribeEventBus = (() => {
|
|
298
|
+
const unsub1 = runEventBus.onChannel("run:state", () => this.invalidateAndRender());
|
|
299
|
+
const unsub2 = runEventBus.onChannel("worker:lifecycle", () => this.invalidateAndRender());
|
|
300
|
+
const unsub3 = runEventBus.onChannel("ui:invalidate", () => this.invalidateAndRender());
|
|
301
|
+
return () => { unsub1(); unsub2(); unsub3(); };
|
|
302
|
+
})();
|
|
298
303
|
}
|
|
299
304
|
|
|
300
305
|
/**
|
package/src/ui/run-event-bus.ts
CHANGED
|
@@ -40,7 +40,7 @@ const RUN_STATE_TYPES = new Set([
|
|
|
40
40
|
"manifest.saved", "task.claimed", "task.unclaimed", "mailbox_updated",
|
|
41
41
|
]);
|
|
42
42
|
const UI_INVALIDATE_TYPES = new Set([
|
|
43
|
-
"effectiveness_changed", "snapshot_stale",
|
|
43
|
+
"effectiveness_changed", "snapshot_stale", "run.cache_invalidated",
|
|
44
44
|
]);
|
|
45
45
|
|
|
46
46
|
/** Classify an event type string into a typed channel. */
|
|
@@ -787,11 +787,55 @@ export function createRunSnapshotCache(cwd: string, options: RunSnapshotCacheOpt
|
|
|
787
787
|
}
|
|
788
788
|
}
|
|
789
789
|
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
790
|
+
// Coalesced eager refresh on event-bus signals. Previously every
|
|
791
|
+
// `run:state` / `worker:lifecycle` event deleted the cache entry, leaving
|
|
792
|
+
// a window where `widget-model.ts: snapshotCache.get(runId)` returned
|
|
793
|
+
// `undefined`. The widget then fell back to `agentsFor(run)` (a disk read
|
|
794
|
+
// with no snapshot.tasks) and rendered the "0/1 done" branch of
|
|
795
|
+
// `widget-renderer.ts:39-41` instead of the "Phase 1/1 default: 0% (0/3)"
|
|
796
|
+
// branch — producing the live flicker between those two progressPart
|
|
797
|
+
// values every render tick. Replacing the delete with a coalesced
|
|
798
|
+
// refreshIfStale keeps the cache populated so the widget always sees the
|
|
799
|
+
// same logical snapshot between stamp changes; multiple events for the
|
|
800
|
+
// same runId within INVAL_COALESCE_MS are batched into one refresh.
|
|
801
|
+
function localRefresh(runId: string): RunUiSnapshot {
|
|
802
|
+
const previous = entries.get(runId);
|
|
803
|
+
const entry = build(runId, previous);
|
|
804
|
+
entries.set(runId, entry);
|
|
805
|
+
evictIfNeeded();
|
|
806
|
+
return entry.snapshot;
|
|
807
|
+
}
|
|
808
|
+
function localRefreshIfStale(runId: string): RunUiSnapshot {
|
|
809
|
+
const previous = entries.get(runId);
|
|
810
|
+
if (!previous) return localRefresh(runId);
|
|
811
|
+
const now = Date.now();
|
|
812
|
+
if (now - previous.loadedAtMs < ttlMs) return touch(runId, previous);
|
|
813
|
+
const stamps = currentStamps(previous);
|
|
814
|
+
if (sameStamps(stamps, previous.stamps)) return touch(runId, previous);
|
|
815
|
+
return localRefresh(runId);
|
|
816
|
+
}
|
|
817
|
+
const pendingRefreshes = new Map<string, ReturnType<typeof setTimeout>>();
|
|
818
|
+
const INVAL_COALESCE_MS = 80;
|
|
819
|
+
const scheduleRefresh = (runId: string): void => {
|
|
820
|
+
const existing = pendingRefreshes.get(runId);
|
|
821
|
+
if (existing) clearTimeout(existing);
|
|
822
|
+
pendingRefreshes.set(runId, setTimeout(() => {
|
|
823
|
+
pendingRefreshes.delete(runId);
|
|
824
|
+
try { localRefreshIfStale(runId); } catch { /* best-effort; widget falls back gracefully */ }
|
|
825
|
+
}, INVAL_COALESCE_MS));
|
|
826
|
+
};
|
|
827
|
+
const unsubState = runEventBus.onChannel("run:state", (event) => {
|
|
828
|
+
if (entries.has(event.runId)) scheduleRefresh(event.runId);
|
|
794
829
|
});
|
|
830
|
+
const unsubLifecycle = runEventBus.onChannel("worker:lifecycle", (event) => {
|
|
831
|
+
if (entries.has(event.runId)) scheduleRefresh(event.runId);
|
|
832
|
+
});
|
|
833
|
+
const unsubscribe = () => {
|
|
834
|
+
unsubState();
|
|
835
|
+
unsubLifecycle();
|
|
836
|
+
for (const timer of pendingRefreshes.values()) clearTimeout(timer);
|
|
837
|
+
pendingRefreshes.clear();
|
|
838
|
+
};
|
|
795
839
|
|
|
796
840
|
return {
|
|
797
841
|
get(runId: string): RunUiSnapshot | undefined {
|
|
@@ -799,20 +843,10 @@ export function createRunSnapshotCache(cwd: string, options: RunSnapshotCacheOpt
|
|
|
799
843
|
return entry ? touch(runId, entry) : undefined;
|
|
800
844
|
},
|
|
801
845
|
refresh(runId: string): RunUiSnapshot {
|
|
802
|
-
|
|
803
|
-
const entry = build(runId, previous);
|
|
804
|
-
entries.set(runId, entry);
|
|
805
|
-
evictIfNeeded();
|
|
806
|
-
return entry.snapshot;
|
|
846
|
+
return localRefresh(runId);
|
|
807
847
|
},
|
|
808
848
|
refreshIfStale(runId: string): RunUiSnapshot {
|
|
809
|
-
|
|
810
|
-
if (!previous) return this.refresh(runId);
|
|
811
|
-
const now = Date.now();
|
|
812
|
-
if (now - previous.loadedAtMs < ttlMs) return touch(runId, previous);
|
|
813
|
-
const stamps = currentStamps(previous);
|
|
814
|
-
if (sameStamps(stamps, previous.stamps)) return touch(runId, previous);
|
|
815
|
-
return this.refresh(runId);
|
|
849
|
+
return localRefreshIfStale(runId);
|
|
816
850
|
},
|
|
817
851
|
preloadStale,
|
|
818
852
|
preloadAllStale,
|