pi-crew 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +153 -0
- package/README.md +17 -1
- package/docs/architecture.md +2 -0
- package/docs/migration-v0.4-v0.5.md +19 -2
- package/docs/pi-crew-v0.5.5-audit-fix-plan.md +133 -0
- package/package.json +7 -5
- package/src/benchmark/benchmark-runner.ts +45 -0
- package/src/benchmark/feedback-loop.ts +5 -0
- package/src/config/config.ts +38 -4
- package/src/config/defaults.ts +5 -0
- package/src/config/suggestions.ts +8 -0
- package/src/extension/async-notifier.ts +10 -1
- package/src/extension/cross-extension-rpc.ts +1 -1
- package/src/extension/notification-router.ts +18 -0
- package/src/extension/register.ts +13 -17
- package/src/extension/registration/subagent-tools.ts +1 -1
- package/src/extension/team-tool/anchor.ts +201 -0
- package/src/extension/team-tool/api.ts +2 -1
- package/src/extension/team-tool/auto-summarize.ts +154 -0
- package/src/extension/team-tool/run.ts +37 -2
- package/src/extension/team-tool.ts +44 -2
- package/src/hooks/registry.ts +1 -3
- package/src/observability/event-bus.ts +13 -4
- package/src/observability/event-to-metric.ts +0 -2
- package/src/runtime/anchor-manager.ts +473 -0
- package/src/runtime/async-runner.ts +8 -4
- package/src/runtime/auto-summarize.ts +350 -0
- package/src/runtime/background-runner.ts +2 -1
- package/src/runtime/budget-tracker.ts +354 -0
- package/src/runtime/chain-runner.ts +507 -0
- package/src/runtime/child-pi.ts +24 -6
- package/src/runtime/crash-recovery.ts +5 -4
- package/src/runtime/crew-agent-records.ts +32 -1
- package/src/runtime/custom-tools/irc-tool.ts +13 -0
- package/src/runtime/custom-tools/submit-result-tool.ts +3 -2
- package/src/runtime/delivery-coordinator.ts +10 -3
- package/src/runtime/dynamic-script-runner.ts +482 -0
- package/src/runtime/handoff-manager.ts +589 -0
- package/src/runtime/hidden-handoff.ts +424 -0
- package/src/runtime/live-agent-manager.ts +20 -4
- package/src/runtime/live-session-runtime.ts +39 -4
- package/src/runtime/manifest-cache.ts +2 -1
- package/src/runtime/model-resolver.ts +16 -4
- package/src/runtime/phase-tracker.ts +373 -0
- package/src/runtime/pipeline-runner.ts +514 -0
- package/src/runtime/retry-runner.ts +354 -0
- package/src/runtime/sandbox.ts +252 -0
- package/src/runtime/scheduler.ts +7 -2
- package/src/runtime/subagent-manager.ts +1 -1
- package/src/runtime/task-graph.ts +11 -1
- package/src/runtime/task-runner.ts +15 -1
- package/src/runtime/team-runner.ts +4 -3
- package/src/schema/team-tool-schema.ts +31 -0
- package/src/skills/discover-skills.ts +5 -0
- package/src/state/active-run-registry.ts +19 -3
- package/src/state/contracts.ts +9 -0
- package/src/state/crew-init.ts +3 -3
- package/src/state/decision-ledger.ts +26 -32
- package/src/state/event-log-rotation.ts +2 -2
- package/src/state/event-log.ts +17 -4
- package/src/state/mailbox.ts +35 -1
- package/src/state/run-cache.ts +18 -8
- package/src/tools/safe-bash-extension.ts +1 -0
- package/src/tools/safe-bash.ts +153 -20
- package/src/ui/overlays/mailbox-detail-overlay.ts +13 -2
- package/src/ui/powerbar-publisher.ts +1 -0
- package/src/ui/transcript-cache.ts +13 -0
- package/src/utils/bm25-search.ts +16 -8
- package/src/utils/env-filter.ts +8 -5
- package/src/utils/redaction.ts +169 -15
- package/src/utils/sse-parser.ts +10 -1
- package/src/worktree/cleanup.ts +6 -1
- package/workflows/chain.workflow.md +252 -0
- package/workflows/pipeline.workflow.md +27 -0
|
@@ -36,6 +36,7 @@ import { registerRunPromise, resolveRunPromise, rejectRunPromise } from "./run-t
|
|
|
36
36
|
import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
37
37
|
import { CrewCancellationError, buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "./cancellation.ts";
|
|
38
38
|
import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
|
|
39
|
+
import { logInternalError } from "../utils/internal-error.ts";
|
|
39
40
|
|
|
40
41
|
export interface ExecuteTeamRunInput {
|
|
41
42
|
manifest: TeamRunManifest;
|
|
@@ -279,7 +280,7 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
279
280
|
resolveRunPromise(manifest.runId, result);
|
|
280
281
|
cleanupUsage();
|
|
281
282
|
// Terminate live agents for this run — agents are done when the run ends.
|
|
282
|
-
void terminateLiveAgentsForRun(manifest.runId, "completed", appendEvent, manifest.eventsPath).catch(() => {});
|
|
283
|
+
void terminateLiveAgentsForRun(manifest.runId, "completed", appendEvent, manifest.eventsPath).catch((error) => logInternalError("team-runner.completed.terminate", error, `runId=${manifest.runId}`));
|
|
283
284
|
|
|
284
285
|
// Emit run completion hook (100% reliable, fire-and-forget)
|
|
285
286
|
crewHooks.emit({ type: "run_completed", timestamp: new Date().toISOString(), runId: manifest.runId, data: { status: result.manifest.status, taskCount: result.tasks.length } });
|
|
@@ -519,7 +520,7 @@ async function executeTeamRunCore(
|
|
|
519
520
|
attemptId: (attempt) => `${manifest.runId}:${task.id}:attempt-${attempt}`,
|
|
520
521
|
onAttemptFailed: (attempt, error, delayMs, info) => {
|
|
521
522
|
lastAttemptId = info.attemptId;
|
|
522
|
-
appendEventAsync(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, attemptId: info.attemptId, delayMs }, metadata: { attemptId: info.attemptId } }).catch(() => {});
|
|
523
|
+
appendEventAsync(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, attemptId: info.attemptId, delayMs }, metadata: { attemptId: info.attemptId } }).catch((error) => logInternalError("team-runner.retry-attempt", error, `taskId=${task.id}`));
|
|
523
524
|
input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe({ runId: manifest.runId, taskId: task.id }, delayMs);
|
|
524
525
|
},
|
|
525
526
|
onRetryGivenUp: (attempts, error, info) => {
|
|
@@ -536,7 +537,7 @@ async function executeTeamRunCore(
|
|
|
536
537
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
537
538
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
538
539
|
const cancelledTasks = freshTasks.map((item) => item.id === task.id && (item.status === "queued" || item.status === "running") ? { ...item, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: `${reason.message} (${reason.code})` } : item);
|
|
539
|
-
appendEventAsync(freshManifest.eventsPath, { type: "task.cancelled", runId: freshManifest.runId, taskId: task.id, message: reason.message, data: { reason, phase: "retry" }, metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined }).catch(() => {});
|
|
540
|
+
appendEventAsync(freshManifest.eventsPath, { type: "task.cancelled", runId: freshManifest.runId, taskId: task.id, message: reason.message, data: { reason, phase: "retry" }, metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined }).catch((error) => logInternalError("team-runner.cancelled", error, `taskId=${task.id}`));
|
|
540
541
|
return { manifest: updateRunStatus(freshManifest, "cancelled", reason.message), tasks: cancelledTasks };
|
|
541
542
|
}
|
|
542
543
|
if (lastFailed) return lastFailed;
|
|
@@ -58,6 +58,7 @@ export const TeamToolParams = Type.Object({
|
|
|
58
58
|
Type.Literal("api"),
|
|
59
59
|
Type.Literal("settings"),
|
|
60
60
|
Type.Literal("steer"),
|
|
61
|
+
Type.Literal("invalidate"),
|
|
61
62
|
Type.Literal("health"),
|
|
62
63
|
Type.Literal("graph"),
|
|
63
64
|
Type.Literal("onboard"),
|
|
@@ -68,6 +69,9 @@ export const TeamToolParams = Type.Object({
|
|
|
68
69
|
Type.Literal("orchestrate"),
|
|
69
70
|
Type.Literal("schedule"),
|
|
70
71
|
Type.Literal("scheduled"),
|
|
72
|
+
Type.Literal("anchor"),
|
|
73
|
+
Type.Literal("auto-summarize"),
|
|
74
|
+
Type.Literal("auto_boomerang"),
|
|
71
75
|
],
|
|
72
76
|
{ description: "Team action. Defaults to 'list' when omitted." },
|
|
73
77
|
),
|
|
@@ -209,6 +213,27 @@ export const TeamToolParams = Type.Object({
|
|
|
209
213
|
description: "Mark certain bash commands as excludeFromContext to reduce context tokens (default: false).",
|
|
210
214
|
}),
|
|
211
215
|
),
|
|
216
|
+
// Budget tracking options
|
|
217
|
+
budgetTotal: Type.Optional(
|
|
218
|
+
Type.Number({
|
|
219
|
+
description: "Total token budget for the run. When set, enables budget tracking with default 80% warning and 95% abort thresholds.",
|
|
220
|
+
minimum: 1,
|
|
221
|
+
}),
|
|
222
|
+
),
|
|
223
|
+
budgetWarning: Type.Optional(
|
|
224
|
+
Type.Number({
|
|
225
|
+
description: "Budget warning threshold as a fraction (0-1). Default: 0.8 (80%). Emits warning event when this threshold is crossed.",
|
|
226
|
+
minimum: 0,
|
|
227
|
+
maximum: 1,
|
|
228
|
+
}),
|
|
229
|
+
),
|
|
230
|
+
budgetAbort: Type.Optional(
|
|
231
|
+
Type.Number({
|
|
232
|
+
description: "Budget abort threshold as a fraction (0-1). Default: 0.95 (95%). Aborts further execution when this threshold is crossed.",
|
|
233
|
+
minimum: 0,
|
|
234
|
+
maximum: 1,
|
|
235
|
+
}),
|
|
236
|
+
),
|
|
212
237
|
});
|
|
213
238
|
|
|
214
239
|
export interface TeamToolParamsValue {
|
|
@@ -294,4 +319,10 @@ export interface TeamToolParamsValue {
|
|
|
294
319
|
once?: string | number;
|
|
295
320
|
/** Mark certain bash commands as excludeFromContext to reduce context tokens (default: false). */
|
|
296
321
|
excludeContextBash?: boolean;
|
|
322
|
+
/** Total token budget for the run. When set, enables budget tracking. */
|
|
323
|
+
budgetTotal?: number;
|
|
324
|
+
/** Budget warning threshold as a fraction (0-1). Default: 0.8. */
|
|
325
|
+
budgetWarning?: number;
|
|
326
|
+
/** Budget abort threshold as a fraction (0-1). Default: 0.95. */
|
|
327
|
+
budgetAbort?: number;
|
|
297
328
|
}
|
|
@@ -6,6 +6,9 @@ import { isSafePathId, resolveContainedPath, resolveRealContainedPath } from "..
|
|
|
6
6
|
|
|
7
7
|
const PACKAGE_SKILLS_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "skills");
|
|
8
8
|
|
|
9
|
+
const CACHE_TTL_MS = 30_000; // 30 seconds
|
|
10
|
+
let cache: { skills: SkillDescriptor[]; cachedAt: number; cwd: string } | null = null;
|
|
11
|
+
|
|
9
12
|
export interface SkillDescriptor {
|
|
10
13
|
name: string;
|
|
11
14
|
description: string;
|
|
@@ -28,6 +31,7 @@ function frontmatterDescription(content: string): string | undefined {
|
|
|
28
31
|
}
|
|
29
32
|
|
|
30
33
|
export function discoverSkills(cwd: string): SkillDescriptor[] {
|
|
34
|
+
if (cache && cache.cwd === cwd && Date.now() - cache.cachedAt < CACHE_TTL_MS) return cache.skills;
|
|
31
35
|
const results: SkillDescriptor[] = [];
|
|
32
36
|
for (const dir of listSkillDirs(cwd)) {
|
|
33
37
|
if (!fs.existsSync(dir.root)) continue;
|
|
@@ -63,5 +67,6 @@ export function discoverSkills(cwd: string): SkillDescriptor[] {
|
|
|
63
67
|
logInternalError("discoverSkills.readdir", error, `root=${dir.root}`);
|
|
64
68
|
}
|
|
65
69
|
}
|
|
70
|
+
cache = { skills: results, cachedAt: Date.now(), cwd };
|
|
66
71
|
return results;
|
|
67
72
|
}
|
|
@@ -10,6 +10,9 @@ import { sharedScanCache } from "../utils/scan-cache.ts";
|
|
|
10
10
|
import { sleepSync } from "../utils/sleep.ts";
|
|
11
11
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
12
12
|
|
|
13
|
+
/** Magic bytes prefix for binary registry to prevent deserialization of hostile files. */
|
|
14
|
+
const BINARY_MAGIC = Buffer.from("PICREW2BIN", "utf-8");
|
|
15
|
+
|
|
13
16
|
export interface ActiveRunRegistryEntry {
|
|
14
17
|
runId: string;
|
|
15
18
|
cwd: string;
|
|
@@ -111,7 +114,11 @@ export function readActiveRunRegistry(maxEntries = DEFAULT_CACHE.manifestMaxEntr
|
|
|
111
114
|
// corrupt; this lets a 2-release migration co-exist with old readers.
|
|
112
115
|
try {
|
|
113
116
|
const buf = fs.readFileSync(registryBinaryPath());
|
|
114
|
-
|
|
117
|
+
// Security: verify magic bytes before deserializing to prevent RCE from hostile files
|
|
118
|
+
if (buf.length < BINARY_MAGIC.length || !buf.slice(0, BINARY_MAGIC.length).equals(BINARY_MAGIC)) {
|
|
119
|
+
throw new Error("Invalid binary registry: missing magic bytes");
|
|
120
|
+
}
|
|
121
|
+
parsed = deserialize(buf.slice(BINARY_MAGIC.length));
|
|
115
122
|
} catch {
|
|
116
123
|
try {
|
|
117
124
|
parsed = JSON.parse(fs.readFileSync(registryPath(), "utf-8"));
|
|
@@ -128,14 +135,23 @@ export function readActiveRunRegistry(maxEntries = DEFAULT_CACHE.manifestMaxEntr
|
|
|
128
135
|
}
|
|
129
136
|
|
|
130
137
|
function writeEntries(entries: ActiveRunRegistryEntry[]): void {
|
|
131
|
-
const
|
|
138
|
+
const max = DEFAULT_CACHE.manifestMaxEntries;
|
|
139
|
+
// FIX: Emit warning when entries overflow the cap, instead of silent drop.
|
|
140
|
+
if (entries.length > max) {
|
|
141
|
+
logInternalError(
|
|
142
|
+
"active-run-registry.overflow",
|
|
143
|
+
new Error(`${entries.length - max} entries dropped (cap=${max})`),
|
|
144
|
+
JSON.stringify({ dropped: entries.length - max, total: entries.length, cap: max }),
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
const trimmed = entries.slice(0, max);
|
|
132
148
|
fs.mkdirSync(path.dirname(registryPath()), { recursive: true });
|
|
133
149
|
// 2.4 — dual-ship: write both formats. Readers prefer binary; legacy
|
|
134
150
|
// readers (other tools / older releases) keep using the JSON file.
|
|
135
151
|
atomicWriteJson(registryPath(), trimmed);
|
|
136
152
|
try {
|
|
137
153
|
const tempBin = `${registryBinaryPath()}.${process.pid}.${Date.now()}.tmp`;
|
|
138
|
-
fs.writeFileSync(tempBin, serialize(trimmed));
|
|
154
|
+
fs.writeFileSync(tempBin, Buffer.concat([BINARY_MAGIC, serialize(trimmed)]));
|
|
139
155
|
fs.renameSync(tempBin, registryBinaryPath());
|
|
140
156
|
} catch (error) {
|
|
141
157
|
logInternalError("active-run-registry.binary-write", error);
|
package/src/state/contracts.ts
CHANGED
|
@@ -67,6 +67,15 @@ export const TEAM_EVENT_TYPES = [
|
|
|
67
67
|
"task.resumed",
|
|
68
68
|
"task.retried",
|
|
69
69
|
"supervisor.contact",
|
|
70
|
+
// Budget tracking events
|
|
71
|
+
"budget.initialized",
|
|
72
|
+
"budget.warning",
|
|
73
|
+
"budget.exhausted",
|
|
74
|
+
// Phase tracking events
|
|
75
|
+
"phase.started",
|
|
76
|
+
"phase.completed",
|
|
77
|
+
"phase.skipped",
|
|
78
|
+
"phase.failed",
|
|
70
79
|
] as const;
|
|
71
80
|
export type TeamEventType = typeof TEAM_EVENT_TYPES[number];
|
|
72
81
|
|
package/src/state/crew-init.ts
CHANGED
|
@@ -108,9 +108,9 @@ export async function ensureCrewDirectory(cwd: string): Promise<void> {
|
|
|
108
108
|
];
|
|
109
109
|
|
|
110
110
|
for (const dir of dirs) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
}
|
|
111
|
+
// Use mkdirSync directly with recursive:true to avoid TOCTOU race.
|
|
112
|
+
// This is atomic and doesn't require existsSync check.
|
|
113
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
114
114
|
}
|
|
115
115
|
|
|
116
116
|
// 2. Create .gitkeep placeholders in directories that should be tracked
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
|
|
2
2
|
import { dirname } from "path";
|
|
3
|
+
import { atomicWriteFile } from "./atomic-write.ts";
|
|
3
4
|
|
|
4
5
|
export interface CoherenceMark {
|
|
5
6
|
matchesPrior: boolean;
|
|
@@ -21,9 +22,12 @@ export interface RolloutEntry {
|
|
|
21
22
|
|
|
22
23
|
/**
|
|
23
24
|
* Get the ledger file path for a given run ID.
|
|
25
|
+
* SECURITY: Accept stateRoot param to use it for path computation
|
|
26
|
+
* instead of hardcoded path, ensuring stateRoot containment.
|
|
24
27
|
*/
|
|
25
|
-
function getLedgerPath(runId: string): string {
|
|
26
|
-
|
|
28
|
+
function getLedgerPath(runId: string, stateRoot?: string): string {
|
|
29
|
+
const base = stateRoot ?? `.crew/state/runs/${runId}`;
|
|
30
|
+
return `${base}/decision-ledger.jsonl`;
|
|
27
31
|
}
|
|
28
32
|
|
|
29
33
|
/**
|
|
@@ -44,19 +48,19 @@ function computeCoherence(entry: RolloutEntry, ledger: RolloutEntry[]): Coherenc
|
|
|
44
48
|
entry.decisionMark === previousEntry.decisionMark ||
|
|
45
49
|
Boolean(entry.priorWinner && entry.topCandidates.includes(entry.priorWinner));
|
|
46
50
|
|
|
47
|
-
// Check last
|
|
48
|
-
const recentEntries = ledger.slice(-
|
|
51
|
+
// Check last 10 entries for recursive pattern
|
|
52
|
+
const recentEntries = ledger.slice(-10);
|
|
49
53
|
const recentDecisions = recentEntries.map((e) => e.decisionMark);
|
|
50
54
|
const currentDecision = entry.decisionMark;
|
|
51
55
|
|
|
52
56
|
const recursiveMatches = recentDecisions.filter((d) => d === currentDecision).length;
|
|
53
|
-
const matchesRecursive = recursiveMatches >= 2;
|
|
57
|
+
const matchesRecursive = recursiveMatches >= Math.ceil(recentDecisions.length / 2); // At least half match
|
|
54
58
|
|
|
55
59
|
const promotionAllowed = matchesPrior || matchesRecursive;
|
|
56
60
|
|
|
57
61
|
let reason: string;
|
|
58
62
|
if (matchesPrior && matchesRecursive) {
|
|
59
|
-
reason = `Matches prior winner and recursive pattern (${recursiveMatches}
|
|
63
|
+
reason = `Matches prior winner and recursive pattern (${recursiveMatches}/${recentDecisions.length} recent decisions)`;
|
|
60
64
|
} else if (matchesPrior) {
|
|
61
65
|
reason = `Matches prior winner decision`;
|
|
62
66
|
} else if (matchesRecursive) {
|
|
@@ -94,17 +98,17 @@ export function initLedger(runId: string): void {
|
|
|
94
98
|
/**
|
|
95
99
|
* Append a new entry to the decision ledger.
|
|
96
100
|
* Automatically computes and adds coherence marks.
|
|
101
|
+
* FIX: Uses atomic write to prevent partial writes on crash.
|
|
97
102
|
*/
|
|
98
103
|
export function appendEntry(runId: string, entry: RolloutEntry): RolloutEntry {
|
|
99
|
-
const ledgerPath = getLedgerPath(runId);
|
|
100
|
-
|
|
101
104
|
// Ensure directory exists
|
|
105
|
+
const ledgerPath = getLedgerPath(runId);
|
|
102
106
|
const dir = dirname(ledgerPath);
|
|
103
107
|
if (!existsSync(dir)) {
|
|
104
108
|
mkdirSync(dir, { recursive: true });
|
|
105
109
|
}
|
|
106
110
|
|
|
107
|
-
// Get existing entries to compute coherence
|
|
111
|
+
// Get existing entries to compute coherence (and use same result for write)
|
|
108
112
|
const ledger = getLedger(runId);
|
|
109
113
|
|
|
110
114
|
// Compute coherence
|
|
@@ -114,9 +118,11 @@ export function appendEntry(runId: string, entry: RolloutEntry): RolloutEntry {
|
|
|
114
118
|
coherenceMark,
|
|
115
119
|
};
|
|
116
120
|
|
|
117
|
-
// Append to JSONL file
|
|
121
|
+
// Append to JSONL file using atomic write to prevent corruption
|
|
122
|
+
// Use the already-loaded ledger content (no double-read)
|
|
118
123
|
const line = JSON.stringify(entryWithCoherence) + "\n";
|
|
119
|
-
|
|
124
|
+
const existingContent = ledger.length > 0 ? ledger.map((e) => JSON.stringify(e)).join("\n") + "\n" : "";
|
|
125
|
+
atomicWriteFile(ledgerPath, existingContent + line);
|
|
120
126
|
return entryWithCoherence;
|
|
121
127
|
}
|
|
122
128
|
|
|
@@ -233,7 +239,7 @@ function overrideLastEntry(runId: string, coherenceMark: import("./types.js").Co
|
|
|
233
239
|
ledger[lastIndex] = { ...ledger[lastIndex], coherenceMark };
|
|
234
240
|
// Rewrite entire ledger to preserve all entries
|
|
235
241
|
const ledgerPath = getLedgerPath(runId);
|
|
236
|
-
|
|
242
|
+
atomicWriteFile(ledgerPath, ledger.map((e) => JSON.stringify(e)).join("\n") + "\n");
|
|
237
243
|
return ledger[lastIndex];
|
|
238
244
|
}
|
|
239
245
|
|
|
@@ -270,28 +276,22 @@ export function promoteCandidate(runId: string, candidate: string): RolloutEntry
|
|
|
270
276
|
coherenceMark,
|
|
271
277
|
};
|
|
272
278
|
|
|
273
|
-
//
|
|
274
|
-
|
|
275
|
-
const lastIndex = ledger.length - 1;
|
|
276
|
-
ledger[lastIndex] = entry;
|
|
277
|
-
} else {
|
|
278
|
-
// No existing entries - just write this one
|
|
279
|
-
ledger.push(entry);
|
|
280
|
-
}
|
|
279
|
+
// Always push new entry (append-only pattern)
|
|
280
|
+
ledger.push(entry);
|
|
281
281
|
|
|
282
|
-
// Rewrite entire ledger to preserve all entries
|
|
282
|
+
// Rewrite entire ledger atomically to preserve all entries
|
|
283
283
|
const ledgerPath = getLedgerPath(runId);
|
|
284
284
|
const dir = dirname(ledgerPath);
|
|
285
285
|
if (!existsSync(dir)) {
|
|
286
286
|
mkdirSync(dir, { recursive: true });
|
|
287
287
|
}
|
|
288
|
-
|
|
288
|
+
atomicWriteFile(ledgerPath, ledger.map((e) => JSON.stringify(e)).join("\n") + "\n");
|
|
289
289
|
|
|
290
290
|
return entry;
|
|
291
291
|
}
|
|
292
292
|
|
|
293
293
|
/**
|
|
294
|
-
* Decay a candidate by marking it as
|
|
294
|
+
* Decay a candidate by marking it as accepted with proper coherence.
|
|
295
295
|
*/
|
|
296
296
|
export function decayCandidate(runId: string, candidate: string): RolloutEntry {
|
|
297
297
|
const latestDecision = getLatestDecision(runId);
|
|
@@ -323,14 +323,8 @@ export function decayCandidate(runId: string, candidate: string): RolloutEntry {
|
|
|
323
323
|
coherenceMark,
|
|
324
324
|
};
|
|
325
325
|
|
|
326
|
-
//
|
|
327
|
-
|
|
328
|
-
const lastIndex = ledger.length - 1;
|
|
329
|
-
ledger[lastIndex] = entry;
|
|
330
|
-
} else {
|
|
331
|
-
// No existing entries - just write this one
|
|
332
|
-
ledger.push(entry);
|
|
333
|
-
}
|
|
326
|
+
// Always push new entry (append-only pattern)
|
|
327
|
+
ledger.push(entry);
|
|
334
328
|
|
|
335
329
|
// Rewrite entire ledger to preserve all entries
|
|
336
330
|
const ledgerPath = getLedgerPath(runId);
|
|
@@ -338,7 +332,7 @@ export function decayCandidate(runId: string, candidate: string): RolloutEntry {
|
|
|
338
332
|
if (!existsSync(dir)) {
|
|
339
333
|
mkdirSync(dir, { recursive: true });
|
|
340
334
|
}
|
|
341
|
-
|
|
335
|
+
atomicWriteFile(ledgerPath, ledger.map((e) => JSON.stringify(e)).join("\n") + "\n");
|
|
342
336
|
|
|
343
337
|
return entry;
|
|
344
338
|
}
|
|
@@ -209,9 +209,9 @@ export function getEventLogStats(eventsPath: string): EventLogStats | undefined
|
|
|
209
209
|
if (newlineCount === 0) firstLineBytes = offset + i + 1;
|
|
210
210
|
newlineCount++;
|
|
211
211
|
}
|
|
212
|
-
}
|
|
213
|
-
offset += bytesRead;
|
|
214
212
|
}
|
|
213
|
+
offset += bytesRead;
|
|
214
|
+
}
|
|
215
215
|
} finally {
|
|
216
216
|
fs.closeSync(scanFd);
|
|
217
217
|
}
|
package/src/state/event-log.ts
CHANGED
|
@@ -167,11 +167,16 @@ function nextSequence(eventsPath: string): number {
|
|
|
167
167
|
if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
|
|
168
168
|
return cached.seq + 1;
|
|
169
169
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
170
|
+
// FIX: Trust the sidecar seq file if it exists and the file is non-empty.
|
|
171
|
+
// Only fall back to O(n) scan if sidecar is missing or file shrunk unexpectedly.
|
|
172
|
+
const stored = readStoredSequence(eventsPath);
|
|
173
|
+
if (stored !== undefined && (!cached || stat.size >= cached.size)) {
|
|
174
|
+
sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq: stored });
|
|
175
|
+
return stored + 1;
|
|
173
176
|
}
|
|
177
|
+
const current = scanSequence(eventsPath);
|
|
174
178
|
sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq: current });
|
|
179
|
+
persistSequence(eventsPath, current);
|
|
175
180
|
return current + 1;
|
|
176
181
|
}
|
|
177
182
|
|
|
@@ -293,7 +298,7 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
293
298
|
}
|
|
294
299
|
return fullEvent;
|
|
295
300
|
});
|
|
296
|
-
asyncQueues.set(queueKey, next.catch(() => {}));
|
|
301
|
+
asyncQueues.set(queueKey, next.catch((error) => { logInternalError("event-log.async-queue", error, eventsPath); asyncQueues.delete(queueKey); }));
|
|
297
302
|
return next;
|
|
298
303
|
}
|
|
299
304
|
|
|
@@ -425,8 +430,16 @@ function flushOneEventLogBuffer(eventsPath: string): void {
|
|
|
425
430
|
bufferedQueues.delete(eventsPath);
|
|
426
431
|
const timer = bufferedTimers.get(eventsPath);
|
|
427
432
|
if (timer) clearTimeout(timer);
|
|
433
|
+
// MEDIUM-13: Delete timer entry only after successful flush (in finally block)
|
|
428
434
|
bufferedTimers.delete(eventsPath);
|
|
429
435
|
if (!queue || queue.length === 0) return;
|
|
436
|
+
|
|
437
|
+
// HIGH-10: Clean up queue if it exceeds limit to prevent unbounded growth
|
|
438
|
+
if (queue.length > 1000) {
|
|
439
|
+
// Keep only the last 500 entries
|
|
440
|
+
queue.splice(0, queue.length - 500);
|
|
441
|
+
}
|
|
442
|
+
|
|
430
443
|
try {
|
|
431
444
|
withEventLogLockSync(eventsPath, () => {
|
|
432
445
|
for (const item of queue) {
|
package/src/state/mailbox.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { redactSecrets } from "../utils/redaction.ts";
|
|
|
6
6
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
7
7
|
import { atomicWriteFile } from "./atomic-write.ts";
|
|
8
8
|
import { withEventLogLockSync } from "./event-log.ts";
|
|
9
|
+
import { DEFAULT_MAILBOX } from "../config/defaults.ts";
|
|
9
10
|
|
|
10
11
|
export type MailboxDirection = "inbox" | "outbox";
|
|
11
12
|
export type MailboxMessageStatus = "queued" | "delivered" | "acknowledged";
|
|
@@ -228,7 +229,7 @@ function safeReadMailboxFile(filePath: string, direction: MailboxDirection): Mai
|
|
|
228
229
|
* primary file. Readers continue to see all messages because
|
|
229
230
|
* `safeReadMailboxFile` walks both the primary file and any archives.
|
|
230
231
|
*/
|
|
231
|
-
const MAILBOX_ARCHIVE_THRESHOLD_BYTES =
|
|
232
|
+
const MAILBOX_ARCHIVE_THRESHOLD_BYTES = DEFAULT_MAILBOX.perFileThresholdBytes;
|
|
232
233
|
function rotateMailboxFileIfNeeded(filePath: string, thresholdBytes = MAILBOX_ARCHIVE_THRESHOLD_BYTES): boolean {
|
|
233
234
|
try {
|
|
234
235
|
if (!fs.existsSync(filePath)) return false;
|
|
@@ -238,6 +239,8 @@ function rotateMailboxFileIfNeeded(filePath: string, thresholdBytes = MAILBOX_AR
|
|
|
238
239
|
const archivePath = `${filePath}.${ts}.archive.jsonl`;
|
|
239
240
|
fs.renameSync(filePath, archivePath);
|
|
240
241
|
fs.writeFileSync(filePath, "", "utf-8");
|
|
242
|
+
// FIX: Prune old archives so total per-direction count stays bounded.
|
|
243
|
+
pruneOldMailboxArchives(filePath);
|
|
241
244
|
return true;
|
|
242
245
|
} catch (error) {
|
|
243
246
|
logInternalError("mailbox.rotate", error, filePath);
|
|
@@ -245,6 +248,27 @@ function rotateMailboxFileIfNeeded(filePath: string, thresholdBytes = MAILBOX_AR
|
|
|
245
248
|
}
|
|
246
249
|
}
|
|
247
250
|
|
|
251
|
+
/**
|
|
252
|
+
* Keep at most `DEFAULT_MAILBOX.maxArchivesPerDirection` archive files per
|
|
253
|
+
* mailbox. Older archives are deleted. Prevents unbounded growth on long runs.
|
|
254
|
+
*/
|
|
255
|
+
function pruneOldMailboxArchives(mailboxFilePath: string): void {
|
|
256
|
+
try {
|
|
257
|
+
const dir = path.dirname(mailboxFilePath);
|
|
258
|
+
const base = path.basename(mailboxFilePath);
|
|
259
|
+
const archives = fs
|
|
260
|
+
.readdirSync(dir)
|
|
261
|
+
.filter((f) => f.startsWith(base) && f.includes(".archive.jsonl"))
|
|
262
|
+
.sort(); // Chronological (ISO timestamp in filename)
|
|
263
|
+
const excess = archives.length - DEFAULT_MAILBOX.maxArchivesPerDirection;
|
|
264
|
+
for (let i = 0; i < excess; i += 1) {
|
|
265
|
+
fs.rmSync(path.join(dir, archives[i]), { force: true });
|
|
266
|
+
}
|
|
267
|
+
} catch (error) {
|
|
268
|
+
logInternalError("mailbox.prune", error, mailboxFilePath);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
248
272
|
export function readMailbox(manifest: TeamRunManifest, direction?: MailboxDirection, taskId?: string, kind?: MailboxMessageKind): MailboxMessage[] {
|
|
249
273
|
const directions = direction ? [direction] : ["inbox", "outbox"] as const;
|
|
250
274
|
return directions.flatMap((item) => safeReadMailboxFile(mailboxFile(manifest, item, taskId), item)).filter((msg) => !kind || msg.kind === kind).sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
@@ -289,6 +313,16 @@ export function readDeliveryState(manifest: TeamRunManifest): MailboxDeliverySta
|
|
|
289
313
|
|
|
290
314
|
function writeDeliveryState(manifest: TeamRunManifest, state: MailboxDeliveryState): void {
|
|
291
315
|
ensureRunMailbox(manifest);
|
|
316
|
+
// Prune oldest entries if capped
|
|
317
|
+
const MAX_DELIVERY_MESSAGES = 10000;
|
|
318
|
+
if (Object.keys(state.messages).length > MAX_DELIVERY_MESSAGES) {
|
|
319
|
+
const sorted = Object.entries(state.messages).sort(([, a], [, b]) => {
|
|
320
|
+
const order = { queued: 0, delivered: 1, acknowledged: 2 };
|
|
321
|
+
return (order[a] ?? 3) - (order[b] ?? 3);
|
|
322
|
+
});
|
|
323
|
+
const trimmed = sorted.slice(0, MAX_DELIVERY_MESSAGES);
|
|
324
|
+
state.messages = Object.fromEntries(trimmed);
|
|
325
|
+
}
|
|
292
326
|
atomicWriteFile(deliveryFile(manifest, true), `${JSON.stringify(redactSecrets(state), null, 2)}\n`);
|
|
293
327
|
}
|
|
294
328
|
|
package/src/state/run-cache.ts
CHANGED
|
@@ -3,6 +3,8 @@ import * as path from "node:path";
|
|
|
3
3
|
import * as crypto from "node:crypto";
|
|
4
4
|
import { projectCrewRoot } from "../utils/paths.ts";
|
|
5
5
|
import type { TeamTaskState } from "./types.ts";
|
|
6
|
+
import { atomicWriteFile } from "./atomic-write.ts";
|
|
7
|
+
import { withFileLockSync } from "./locks.ts";
|
|
6
8
|
|
|
7
9
|
const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
|
|
8
10
|
|
|
@@ -31,6 +33,7 @@ export function computeRunCacheKey(goal: string, team: string, workflow: string,
|
|
|
31
33
|
.update(normalized)
|
|
32
34
|
.update(team)
|
|
33
35
|
.update(workflow)
|
|
36
|
+
.update(_cwd)
|
|
34
37
|
.digest("hex")
|
|
35
38
|
.slice(0, 16);
|
|
36
39
|
}
|
|
@@ -61,12 +64,15 @@ export function getCachedRun(cwd: string, cacheKey: string): CacheEntry | null {
|
|
|
61
64
|
const entry = JSON.parse(fs.readFileSync(entryPath, "utf-8")) as CacheEntry;
|
|
62
65
|
|
|
63
66
|
if (Date.now() > entry.expiresAt) {
|
|
64
|
-
// Remove expired entry
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
// Remove expired entry — use lock + atomic write to prevent index corruption
|
|
68
|
+
withFileLockSync(indexPath, () => {
|
|
69
|
+
try {
|
|
70
|
+
fs.unlinkSync(entryPath);
|
|
71
|
+
} catch { /* ignore */ }
|
|
72
|
+
const updatedIndex = JSON.parse(fs.readFileSync(indexPath, "utf-8")) as CacheIndex;
|
|
73
|
+
delete updatedIndex[cacheKey];
|
|
74
|
+
atomicWriteFile(indexPath, JSON.stringify(updatedIndex));
|
|
75
|
+
});
|
|
70
76
|
return null;
|
|
71
77
|
}
|
|
72
78
|
|
|
@@ -109,14 +115,18 @@ export function saveRunToCache(
|
|
|
109
115
|
const entryPath = path.join(dir, `${cacheKey}.json`);
|
|
110
116
|
fs.writeFileSync(entryPath, JSON.stringify(entry), "utf-8");
|
|
111
117
|
|
|
112
|
-
// Update index
|
|
118
|
+
// Update index with atomic write: write to temp file then rename
|
|
113
119
|
const indexPath = path.join(dir, "index.json");
|
|
114
120
|
const index: CacheIndex = fs.existsSync(indexPath)
|
|
115
121
|
? JSON.parse(fs.readFileSync(indexPath, "utf-8"))
|
|
116
122
|
: {};
|
|
117
123
|
|
|
118
124
|
index[cacheKey] = entryPath;
|
|
119
|
-
|
|
125
|
+
|
|
126
|
+
// Atomic write: write to temp file first, then rename
|
|
127
|
+
const tempPath = path.join(dir, "index.json.tmp");
|
|
128
|
+
fs.writeFileSync(tempPath, JSON.stringify(index), "utf-8");
|
|
129
|
+
fs.renameSync(tempPath, indexPath);
|
|
120
130
|
}
|
|
121
131
|
|
|
122
132
|
/**
|
|
@@ -68,6 +68,7 @@ export default function safeBashExtension(pi: ExtensionAPI): void {
|
|
|
68
68
|
"Execute a bash command safely. Blocks dangerous commands like `rm -rf /`, `sudo`, `curl | sh`, etc.",
|
|
69
69
|
parameters: Type.Object({
|
|
70
70
|
command: Type.String({ description: "Bash command to execute" }),
|
|
71
|
+
/** Timeout in seconds (optional). Default: no timeout. If exceeded, the command is killed. */
|
|
71
72
|
timeout: Type.Optional(
|
|
72
73
|
Type.Number({ description: "Timeout in seconds (optional)" }),
|
|
73
74
|
),
|