pi-crew 0.5.1 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +1 -1
- package/docs/actions-reference.md +87 -0
- package/docs/bugs/cross-session-notification-leakage.md +82 -0
- package/docs/coding-agent-optimization.md +268 -0
- package/docs/commands-reference.md +5 -0
- package/docs/deep-review-report.md +384 -0
- package/docs/distillation/cybersecurity-patterns.md +294 -0
- package/docs/migration-v0.4-v0.5.md +191 -0
- package/docs/optimization-plan.md +642 -0
- package/docs/pi-crew-bugs.md +6 -0
- package/docs/pi-mono-opportunities.md +969 -0
- package/docs/pi-mono-review.md +291 -0
- package/{skills → docs/skills}/REFERENCE.md +13 -5
- package/index.ts +1 -1
- package/package.json +19 -16
- package/skills/artifact-analysis-loop/SKILL.md +302 -0
- package/skills/async-worker-recovery/SKILL.md +19 -1
- package/skills/child-pi-spawning/SKILL.md +19 -6
- package/skills/context-artifact-hygiene/SKILL.md +19 -2
- package/skills/delegation-patterns/SKILL.md +68 -3
- package/skills/detection-pipeline-design/SKILL.md +285 -0
- package/skills/event-log-tracing/SKILL.md +20 -6
- package/skills/git-master/SKILL.md +20 -6
- package/skills/hunting-investigation-loop/SKILL.md +401 -0
- package/skills/incident-playbook-construction/SKILL.md +383 -0
- package/skills/live-agent-lifecycle/SKILL.md +20 -6
- package/skills/mailbox-interactive/SKILL.md +19 -6
- package/skills/model-routing-context/SKILL.md +19 -1
- package/skills/multi-perspective-review/SKILL.md +19 -4
- package/skills/observability-reliability/SKILL.md +19 -2
- package/skills/orchestration/SKILL.md +20 -2
- package/skills/ownership-session-security/SKILL.md +20 -2
- package/skills/pi-extension-lifecycle/SKILL.md +20 -2
- package/skills/post-mortem/SKILL.md +7 -2
- package/skills/read-only-explorer/SKILL.md +20 -6
- package/skills/requirements-to-task-packet/SKILL.md +23 -3
- package/skills/resource-discovery-config/SKILL.md +20 -2
- package/skills/runtime-state-reader/SKILL.md +20 -2
- package/skills/safe-bash/SKILL.md +21 -6
- package/skills/scrutinize/SKILL.md +20 -2
- package/skills/secure-agent-orchestration-review/SKILL.md +29 -2
- package/skills/security-review/SKILL.md +560 -0
- package/skills/state-mutation-locking/SKILL.md +22 -2
- package/skills/systematic-debugging/SKILL.md +8 -6
- package/skills/threat-hypothesis-framework/SKILL.md +175 -0
- package/skills/ui-render-performance/SKILL.md +20 -2
- package/skills/verification-before-done/SKILL.md +17 -2
- package/skills/widget-rendering/SKILL.md +21 -6
- package/skills/workspace-isolation/SKILL.md +20 -6
- package/skills/worktree-isolation/SKILL.md +20 -6
- package/src/agents/agent-config.ts +40 -1
- package/src/benchmark/benchmark-runner.ts +245 -0
- package/src/benchmark/feedback-loop.ts +66 -0
- package/src/config/config.ts +22 -5
- package/src/config/role-tools.ts +82 -0
- package/src/config/types.ts +4 -0
- package/src/extension/async-notifier.ts +1 -1
- package/src/extension/autonomous-policy.ts +1 -1
- package/src/extension/crew-cleanup.ts +114 -0
- package/src/extension/cross-extension-rpc.ts +1 -1
- package/src/extension/plan-orchestrate.ts +322 -0
- package/src/extension/register.ts +46 -44
- package/src/extension/registration/command-utils.ts +1 -1
- package/src/extension/registration/commands.ts +1 -1
- package/src/extension/registration/compaction-guard.ts +1 -1
- package/src/extension/registration/subagent-helpers.ts +1 -1
- package/src/extension/registration/subagent-tools.ts +1 -1
- package/src/extension/registration/team-tool.ts +1 -1
- package/src/extension/registration/viewers.ts +1 -1
- package/src/extension/session-summary.ts +1 -1
- package/src/extension/team-manager-command.ts +1 -1
- package/src/extension/team-tool/context.ts +1 -1
- package/src/extension/team-tool/handle-schedule.ts +183 -0
- package/src/extension/team-tool/orchestrate.ts +102 -0
- package/src/extension/team-tool/run.ts +222 -35
- package/src/extension/team-tool.ts +10 -0
- package/src/extension/tool-result.ts +1 -1
- package/src/i18n.ts +1 -1
- package/src/observability/event-bus.ts +60 -0
- package/src/observability/event-to-metric.ts +1 -1
- package/src/prompt/prompt-runtime.ts +1 -1
- package/src/runtime/background-runner.ts +35 -7
- package/src/runtime/child-pi.ts +122 -34
- package/src/runtime/crash-recovery.ts +1 -1
- package/src/runtime/crew-agent-runtime.ts +1 -0
- package/src/runtime/crew-hooks.ts +240 -0
- package/src/runtime/custom-tools/irc-tool.ts +1 -1
- package/src/runtime/custom-tools/submit-result-tool.ts +1 -1
- package/src/runtime/diagnostic-export.ts +38 -2
- package/src/runtime/foreground-control.ts +87 -17
- package/src/runtime/foreground-watchdog.ts +1 -1
- package/src/runtime/live-session-runtime.ts +1 -1
- package/src/runtime/mcp-proxy.ts +1 -1
- package/src/runtime/pi-args.ts +11 -1
- package/src/runtime/pi-json-output.ts +31 -0
- package/src/runtime/pi-spawn.ts +20 -4
- package/src/runtime/process-status.ts +15 -2
- package/src/runtime/progress-tracker.ts +124 -0
- package/src/runtime/runtime-resolver.ts +1 -1
- package/src/runtime/session-resources.ts +1 -1
- package/src/runtime/skill-effectiveness.ts +473 -0
- package/src/runtime/skill-instructions.ts +37 -3
- package/src/runtime/task-runner.ts +122 -18
- package/src/runtime/team-runner.ts +17 -11
- package/src/runtime/tool-progress.ts +10 -3
- package/src/runtime/verification-gates.ts +367 -0
- package/src/schema/team-tool-schema.ts +31 -1
- package/src/state/crew-init.ts +56 -38
- package/src/state/decision-ledger.ts +344 -0
- package/src/state/event-log.ts +136 -10
- package/src/state/hook-instinct-bridge.ts +90 -0
- package/src/state/hook-integrations.ts +51 -0
- package/src/state/instinct-store.ts +249 -0
- package/src/state/run-metrics.ts +135 -0
- package/src/state/state-store.ts +3 -1
- package/src/state/tiered-eval.ts +471 -0
- package/src/state/types-eval.ts +58 -0
- package/src/state/types.ts +7 -0
- package/src/tools/safe-bash-extension.ts +5 -5
- package/src/types/new-api-types.ts +34 -0
- package/src/ui/agent-management-overlay.ts +5 -1
- package/src/ui/crew-widget.ts +30 -16
- package/src/ui/pi-ui-compat.ts +1 -1
- package/src/ui/powerbar-publisher.ts +100 -7
- package/src/ui/run-action-dispatcher.ts +1 -1
- package/src/ui/tool-render.ts +17 -17
- package/src/utils/project-detector.ts +160 -0
- package/src/utils/session-utils.ts +52 -0
- package/src/worktree/worktree-manager.ts +32 -13
- package/test-bugs-all.mjs +1 -1
- package/skills/.gitkeep +0 -0
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
|
|
2
|
+
import { dirname } from "path";
|
|
3
|
+
|
|
4
|
+
export interface CoherenceMark {
|
|
5
|
+
matchesPrior: boolean;
|
|
6
|
+
matchesRecursive: boolean;
|
|
7
|
+
promotionAllowed: boolean;
|
|
8
|
+
reason: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface RolloutEntry {
|
|
12
|
+
rolloutId: string;
|
|
13
|
+
timestamp: string;
|
|
14
|
+
priorWinner?: string;
|
|
15
|
+
searchSpace: string;
|
|
16
|
+
trialCount: number;
|
|
17
|
+
topCandidates: string[];
|
|
18
|
+
decisionMark: "accept" | "watch" | "reject" | "decay";
|
|
19
|
+
coherenceMark: CoherenceMark;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Get the ledger file path for a given run ID.
|
|
24
|
+
*/
|
|
25
|
+
function getLedgerPath(runId: string): string {
|
|
26
|
+
return `.crew/state/runs/${runId}/decision-ledger.jsonl`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Compute coherence marks based on existing ledger entries.
|
|
31
|
+
*/
|
|
32
|
+
function computeCoherence(entry: RolloutEntry, ledger: RolloutEntry[]): CoherenceMark {
|
|
33
|
+
if (ledger.length === 0) {
|
|
34
|
+
return {
|
|
35
|
+
matchesPrior: false,
|
|
36
|
+
matchesRecursive: false,
|
|
37
|
+
promotionAllowed: true,
|
|
38
|
+
reason: "No prior entries - first rollout, promotion allowed",
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const previousEntry = ledger[ledger.length - 1];
|
|
43
|
+
const matchesPrior: boolean =
|
|
44
|
+
entry.decisionMark === previousEntry.decisionMark ||
|
|
45
|
+
Boolean(entry.priorWinner && entry.topCandidates.includes(entry.priorWinner));
|
|
46
|
+
|
|
47
|
+
// Check last 3 entries for recursive pattern
|
|
48
|
+
const recentEntries = ledger.slice(-3);
|
|
49
|
+
const recentDecisions = recentEntries.map((e) => e.decisionMark);
|
|
50
|
+
const currentDecision = entry.decisionMark;
|
|
51
|
+
|
|
52
|
+
const recursiveMatches = recentDecisions.filter((d) => d === currentDecision).length;
|
|
53
|
+
const matchesRecursive = recursiveMatches >= 2;
|
|
54
|
+
|
|
55
|
+
const promotionAllowed = matchesPrior || matchesRecursive;
|
|
56
|
+
|
|
57
|
+
let reason: string;
|
|
58
|
+
if (matchesPrior && matchesRecursive) {
|
|
59
|
+
reason = `Matches prior winner and recursive pattern (${recursiveMatches}/3 recent decisions)`;
|
|
60
|
+
} else if (matchesPrior) {
|
|
61
|
+
reason = `Matches prior winner decision`;
|
|
62
|
+
} else if (matchesRecursive) {
|
|
63
|
+
reason = `Matches recursive pattern (${recursiveMatches}/3 recent decisions)`;
|
|
64
|
+
} else {
|
|
65
|
+
reason = `No match with prior or recursive pattern - requires human review`;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return {
|
|
69
|
+
matchesPrior,
|
|
70
|
+
matchesRecursive,
|
|
71
|
+
promotionAllowed,
|
|
72
|
+
reason,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Initialize a new decision ledger for a run.
|
|
78
|
+
* Creates the directory and ledger file if they don't exist.
|
|
79
|
+
*/
|
|
80
|
+
export function initLedger(runId: string): void {
|
|
81
|
+
const ledgerPath = getLedgerPath(runId);
|
|
82
|
+
const dir = dirname(ledgerPath);
|
|
83
|
+
|
|
84
|
+
if (!existsSync(dir)) {
|
|
85
|
+
mkdirSync(dir, { recursive: true });
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Create empty file if it doesn't exist
|
|
89
|
+
if (!existsSync(ledgerPath)) {
|
|
90
|
+
writeFileSync(ledgerPath, "", "utf-8");
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Append a new entry to the decision ledger.
|
|
96
|
+
* Automatically computes and adds coherence marks.
|
|
97
|
+
*/
|
|
98
|
+
export function appendEntry(runId: string, entry: RolloutEntry): RolloutEntry {
|
|
99
|
+
const ledgerPath = getLedgerPath(runId);
|
|
100
|
+
|
|
101
|
+
// Ensure directory exists
|
|
102
|
+
const dir = dirname(ledgerPath);
|
|
103
|
+
if (!existsSync(dir)) {
|
|
104
|
+
mkdirSync(dir, { recursive: true });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Get existing entries to compute coherence
|
|
108
|
+
const ledger = getLedger(runId);
|
|
109
|
+
|
|
110
|
+
// Compute coherence
|
|
111
|
+
const coherenceMark = computeCoherence(entry, ledger);
|
|
112
|
+
const entryWithCoherence: RolloutEntry = {
|
|
113
|
+
...entry,
|
|
114
|
+
coherenceMark,
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
// Append to JSONL file
|
|
118
|
+
const line = JSON.stringify(entryWithCoherence) + "\n";
|
|
119
|
+
writeFileSync(ledgerPath, line, { flag: "a", encoding: "utf-8" });
|
|
120
|
+
return entryWithCoherence;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Read all entries from the decision ledger.
|
|
125
|
+
*/
|
|
126
|
+
export function getLedger(runId: string): RolloutEntry[] {
|
|
127
|
+
const ledgerPath = getLedgerPath(runId);
|
|
128
|
+
|
|
129
|
+
if (!existsSync(ledgerPath)) {
|
|
130
|
+
return [];
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const content = readFileSync(ledgerPath, "utf-8");
|
|
134
|
+
if (!content.trim()) {
|
|
135
|
+
return [];
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return content
|
|
139
|
+
.split("\n")
|
|
140
|
+
.filter((line) => line.trim())
|
|
141
|
+
.map((line) => JSON.parse(line) as RolloutEntry);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Get the most recent entry from the decision ledger.
|
|
146
|
+
*/
|
|
147
|
+
export function getLatestDecision(runId: string): RolloutEntry | null {
|
|
148
|
+
const ledger = getLedger(runId);
|
|
149
|
+
if (ledger.length === 0) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
return ledger[ledger.length - 1];
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Generate a human-readable markdown summary of the ledger.
|
|
157
|
+
*/
|
|
158
|
+
export function summarizeLedger(runId: string): string {
|
|
159
|
+
const ledger = getLedger(runId);
|
|
160
|
+
|
|
161
|
+
if (ledger.length === 0) {
|
|
162
|
+
return "# Decision Ledger Summary\n\n*No entries recorded yet.*";
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const lines: string[] = [
|
|
166
|
+
"# Decision Ledger Summary",
|
|
167
|
+
"",
|
|
168
|
+
`Run ID: ${runId}`,
|
|
169
|
+
`Total Entries: ${ledger.length}`,
|
|
170
|
+
"",
|
|
171
|
+
"## Entries",
|
|
172
|
+
"",
|
|
173
|
+
];
|
|
174
|
+
|
|
175
|
+
for (let i = 0; i < ledger.length; i++) {
|
|
176
|
+
const entry = ledger[i];
|
|
177
|
+
lines.push(`### ${i + 1}. ${entry.rolloutId}`);
|
|
178
|
+
lines.push("");
|
|
179
|
+
lines.push(`- **Timestamp**: ${entry.timestamp}`);
|
|
180
|
+
lines.push(`- **Search Space**: ${entry.searchSpace}`);
|
|
181
|
+
lines.push(`- **Trial Count**: ${entry.trialCount}`);
|
|
182
|
+
lines.push(`- **Decision**: ${entry.decisionMark}`);
|
|
183
|
+
|
|
184
|
+
if (entry.priorWinner) {
|
|
185
|
+
lines.push(`- **Prior Winner**: ${entry.priorWinner}`);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
lines.push(`- **Top Candidates**: ${entry.topCandidates.join(", ") || "(none)"}`);
|
|
189
|
+
lines.push("");
|
|
190
|
+
lines.push("#### Coherence");
|
|
191
|
+
lines.push(`- **Matches Prior**: ${entry.coherenceMark.matchesPrior ? "✓" : "✗"}`);
|
|
192
|
+
lines.push(`- **Matches Recursive**: ${entry.coherenceMark.matchesRecursive ? "✓" : "✗"}`);
|
|
193
|
+
lines.push(`- **Promotion Allowed**: ${entry.coherenceMark.promotionAllowed ? "✓" : "✗"}`);
|
|
194
|
+
lines.push(`- **Reason**: ${entry.coherenceMark.reason}`);
|
|
195
|
+
lines.push("");
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Summary statistics
|
|
199
|
+
const decisions = ledger.map((e) => e.decisionMark);
|
|
200
|
+
const acceptCount = decisions.filter((d) => d === "accept").length;
|
|
201
|
+
const watchCount = decisions.filter((d) => d === "watch").length;
|
|
202
|
+
const rejectCount = decisions.filter((d) => d === "reject").length;
|
|
203
|
+
const decayCount = decisions.filter((d) => d === "decay").length;
|
|
204
|
+
|
|
205
|
+
lines.push("## Summary");
|
|
206
|
+
lines.push("");
|
|
207
|
+
lines.push(`| Decision | Count |`);
|
|
208
|
+
lines.push(`|----------|-------|`);
|
|
209
|
+
lines.push(`| Accept | ${acceptCount} |`);
|
|
210
|
+
lines.push(`| Watch | ${watchCount} |`);
|
|
211
|
+
lines.push(`| Reject | ${rejectCount} |`);
|
|
212
|
+
lines.push(`| Decay | ${decayCount} |`);
|
|
213
|
+
lines.push("");
|
|
214
|
+
|
|
215
|
+
const promotedCount = ledger.filter((e) => e.coherenceMark.promotionAllowed).length;
|
|
216
|
+
lines.push(`**Promotion Rate**: ${promotedCount}/${ledger.length} (${((promotedCount / ledger.length) * 100).toFixed(1)}%)`);
|
|
217
|
+
|
|
218
|
+
return lines.join("\n");
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Override the coherence mark of the last entry in the ledger.
|
|
223
|
+
* FIX: This preserves all previous entries while updating just the last one.
|
|
224
|
+
* Previously this would truncate the entire ledger!
|
|
225
|
+
*/
|
|
226
|
+
function overrideLastEntry(runId: string, coherenceMark: import("./types.js").CoherenceMark): RolloutEntry {
|
|
227
|
+
const ledger = getLedger(runId);
|
|
228
|
+
if (ledger.length === 0) {
|
|
229
|
+
throw new Error(`No ledger entries found for run ${runId}`);
|
|
230
|
+
}
|
|
231
|
+
// Update the last entry with the new coherence mark
|
|
232
|
+
const lastIndex = ledger.length - 1;
|
|
233
|
+
ledger[lastIndex] = { ...ledger[lastIndex], coherenceMark };
|
|
234
|
+
// Rewrite entire ledger to preserve all entries
|
|
235
|
+
const ledgerPath = getLedgerPath(runId);
|
|
236
|
+
writeFileSync(ledgerPath, ledger.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf-8");
|
|
237
|
+
return ledger[lastIndex];
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Promote a candidate by marking it as accepted with proper coherence.
|
|
242
|
+
*/
|
|
243
|
+
export function promoteCandidate(runId: string, candidate: string): RolloutEntry {
|
|
244
|
+
const latestDecision = getLatestDecision(runId);
|
|
245
|
+
|
|
246
|
+
// Get existing entries to compute proper coherence
|
|
247
|
+
const ledger = getLedger(runId);
|
|
248
|
+
|
|
249
|
+
// Create entry without coherence first
|
|
250
|
+
const entryWithoutCoherence = {
|
|
251
|
+
rolloutId: `promote-${Date.now()}`,
|
|
252
|
+
timestamp: new Date().toISOString(),
|
|
253
|
+
priorWinner: latestDecision?.topCandidates[0],
|
|
254
|
+
searchSpace: latestDecision?.searchSpace || "unknown",
|
|
255
|
+
trialCount: (latestDecision?.trialCount || 0) + 1,
|
|
256
|
+
topCandidates: [candidate],
|
|
257
|
+
decisionMark: "accept" as const,
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
// Compute coherence (empty ledger = no matches)
|
|
261
|
+
const coherenceMark = computeCoherence(entryWithoutCoherence as RolloutEntry, ledger);
|
|
262
|
+
|
|
263
|
+
// Manual promotion always allows further promotion
|
|
264
|
+
coherenceMark.promotionAllowed = true;
|
|
265
|
+
coherenceMark.reason = "Manual promotion - promotion allowed";
|
|
266
|
+
|
|
267
|
+
// Create full entry with coherence
|
|
268
|
+
const entry: RolloutEntry = {
|
|
269
|
+
...entryWithoutCoherence,
|
|
270
|
+
coherenceMark,
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
// Update last entry in memory if there are existing entries
|
|
274
|
+
if (ledger.length > 0) {
|
|
275
|
+
const lastIndex = ledger.length - 1;
|
|
276
|
+
ledger[lastIndex] = entry;
|
|
277
|
+
} else {
|
|
278
|
+
// No existing entries - just write this one
|
|
279
|
+
ledger.push(entry);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Rewrite entire ledger to preserve all entries
|
|
283
|
+
const ledgerPath = getLedgerPath(runId);
|
|
284
|
+
const dir = dirname(ledgerPath);
|
|
285
|
+
if (!existsSync(dir)) {
|
|
286
|
+
mkdirSync(dir, { recursive: true });
|
|
287
|
+
}
|
|
288
|
+
writeFileSync(ledgerPath, ledger.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf-8");
|
|
289
|
+
|
|
290
|
+
return entry;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Decay a candidate by marking it as decayed with proper coherence.
|
|
295
|
+
*/
|
|
296
|
+
export function decayCandidate(runId: string, candidate: string): RolloutEntry {
|
|
297
|
+
const latestDecision = getLatestDecision(runId);
|
|
298
|
+
|
|
299
|
+
// Get existing entries to compute proper coherence
|
|
300
|
+
const ledger = getLedger(runId);
|
|
301
|
+
|
|
302
|
+
// Create entry without coherence first
|
|
303
|
+
const entryWithoutCoherence = {
|
|
304
|
+
rolloutId: `decay-${Date.now()}`,
|
|
305
|
+
timestamp: new Date().toISOString(),
|
|
306
|
+
priorWinner: latestDecision?.topCandidates[0],
|
|
307
|
+
searchSpace: latestDecision?.searchSpace || "unknown",
|
|
308
|
+
trialCount: (latestDecision?.trialCount || 0) + 1,
|
|
309
|
+
topCandidates: [candidate],
|
|
310
|
+
decisionMark: "decay" as const,
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
// Compute coherence (empty ledger = no matches)
|
|
314
|
+
const coherenceMark = computeCoherence(entryWithoutCoherence as RolloutEntry, ledger);
|
|
315
|
+
|
|
316
|
+
// Manual decay never allows promotion
|
|
317
|
+
coherenceMark.promotionAllowed = false;
|
|
318
|
+
coherenceMark.reason = "Manual decay - promotion not allowed";
|
|
319
|
+
|
|
320
|
+
// Create full entry with coherence
|
|
321
|
+
const entry: RolloutEntry = {
|
|
322
|
+
...entryWithoutCoherence,
|
|
323
|
+
coherenceMark,
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
// Update last entry in memory if there are existing entries
|
|
327
|
+
if (ledger.length > 0) {
|
|
328
|
+
const lastIndex = ledger.length - 1;
|
|
329
|
+
ledger[lastIndex] = entry;
|
|
330
|
+
} else {
|
|
331
|
+
// No existing entries - just write this one
|
|
332
|
+
ledger.push(entry);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// Rewrite entire ledger to preserve all entries
|
|
336
|
+
const ledgerPath = getLedgerPath(runId);
|
|
337
|
+
const dir = dirname(ledgerPath);
|
|
338
|
+
if (!existsSync(dir)) {
|
|
339
|
+
mkdirSync(dir, { recursive: true });
|
|
340
|
+
}
|
|
341
|
+
writeFileSync(ledgerPath, ledger.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf-8");
|
|
342
|
+
|
|
343
|
+
return entry;
|
|
344
|
+
}
|
package/src/state/event-log.ts
CHANGED
|
@@ -63,12 +63,17 @@ let appendCounter = 0;
|
|
|
63
63
|
|
|
64
64
|
/** Simple cross-process lock for an eventsPath to prevent JSONL interleave on concurrent append.
|
|
65
65
|
* Detects stale locks by checking the owner PID written inside the lock directory.
|
|
66
|
+
*
|
|
67
|
+
* @deprecated Prefer `appendEventAsync()` for callers in async contexts. The sync lock
|
|
68
|
+
* uses `sleepSync` which blocks the event loop and prevents AbortSignal handlers from firing.
|
|
66
69
|
*/
|
|
67
70
|
export function withEventLogLockSync<T>(eventsPath: string, fn: () => T): T {
|
|
71
|
+
// Ensure parent directory exists before attempting lock
|
|
72
|
+
fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
|
|
68
73
|
const lockDir = `${eventsPath}.lock`;
|
|
69
74
|
const pidFile = path.join(lockDir, "pid");
|
|
70
75
|
const start = Date.now();
|
|
71
|
-
const timeout =
|
|
76
|
+
const timeout = 120000; // 120s timeout for slow CI environments
|
|
72
77
|
const staleMs = 10000;
|
|
73
78
|
let acquired = false;
|
|
74
79
|
while (true) {
|
|
@@ -79,6 +84,8 @@ export function withEventLogLockSync<T>(eventsPath: string, fn: () => T): T {
|
|
|
79
84
|
break;
|
|
80
85
|
} catch {
|
|
81
86
|
if (Date.now() - start > timeout) {
|
|
87
|
+
// Log error and continue without lock — lock is held by live process.
|
|
88
|
+
// Stale detection will clean up dead locks on next attempt.
|
|
82
89
|
logInternalError("event-log.lock-timeout", new Error(`Event log lock timeout for ${eventsPath}`), `lockDir=${lockDir}`);
|
|
83
90
|
break;
|
|
84
91
|
}
|
|
@@ -112,9 +119,15 @@ export function withEventLogLockSync<T>(eventsPath: string, fn: () => T): T {
|
|
|
112
119
|
}
|
|
113
120
|
}
|
|
114
121
|
|
|
115
|
-
function
|
|
116
|
-
|
|
117
|
-
|
|
122
|
+
function evictOldestSequenceCacheEntries(): void {
|
|
123
|
+
// Batch evict oldest 50% of entries when cache is full
|
|
124
|
+
const toEvict = Math.ceil(MAX_SEQUENCE_CACHE_ENTRIES / 2);
|
|
125
|
+
let evicted = 0;
|
|
126
|
+
for (const key of sequenceCache.keys()) {
|
|
127
|
+
if (evicted >= toEvict) break;
|
|
128
|
+
sequenceCache.delete(key);
|
|
129
|
+
evicted++;
|
|
130
|
+
}
|
|
118
131
|
}
|
|
119
132
|
|
|
120
133
|
export function sequencePath(eventsPath: string): string {
|
|
@@ -174,10 +187,116 @@ export function computeEventFingerprint(event: Pick<TeamEvent, "type" | "runId"
|
|
|
174
187
|
return createHash("sha256").update(JSON.stringify({ type: event.type, runId: event.runId, taskId: event.taskId, data: event.data ?? null })).digest("hex").slice(0, 16);
|
|
175
188
|
}
|
|
176
189
|
|
|
190
|
+
/**
|
|
191
|
+
* @deprecated Prefer `appendEventAsync()` in async contexts. The sync lock uses
|
|
192
|
+
* `sleepSync` which blocks the Node.js event loop, preventing AbortSignal handlers
|
|
193
|
+
* from firing and degrading live-agent responsiveness.
|
|
194
|
+
*/
|
|
177
195
|
export function appendEvent(eventsPath: string, event: AppendTeamEvent): TeamEvent {
|
|
178
196
|
return withEventLogLockSync(eventsPath, () => appendEventInsideLock(eventsPath, event));
|
|
179
197
|
}
|
|
180
198
|
|
|
199
|
+
// --- Async write queue (non-blocking alternative to withEventLogLockSync) ---
|
|
200
|
+
const asyncQueues = new Map<string, Promise<unknown>>();
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Append an event to the event log using non-blocking async I/O.
|
|
204
|
+
*
|
|
205
|
+
* Uses a per-eventsPath promise-chain queue to ensure sequential writes without
|
|
206
|
+
* blocking the Node.js event loop. This allows AbortSignal handlers and other
|
|
207
|
+
* async operations to proceed while events are being persisted.
|
|
208
|
+
*
|
|
209
|
+
* For callers that are already in an async context (team-runner, task-runner,
|
|
210
|
+
* foreground-control, etc.), prefer this over the sync `appendEvent()`.
|
|
211
|
+
*/
|
|
212
|
+
export async function appendEventAsync(eventsPath: string, event: AppendTeamEvent): Promise<TeamEvent> {
|
|
213
|
+
const queueKey = eventsPath;
|
|
214
|
+
const prev = asyncQueues.get(queueKey) ?? Promise.resolve();
|
|
215
|
+
const next = prev.then(async (): Promise<TeamEvent> => {
|
|
216
|
+
// Ensure directory exists
|
|
217
|
+
await fs.promises.mkdir(path.dirname(eventsPath), { recursive: true });
|
|
218
|
+
|
|
219
|
+
// Build metadata (same logic as appendEventInsideLock)
|
|
220
|
+
const baseMetadata = event.metadata;
|
|
221
|
+
let metadata: TeamEventMetadata = {
|
|
222
|
+
seq: baseMetadata?.seq ?? nextSequence(eventsPath),
|
|
223
|
+
provenance: baseMetadata?.provenance ?? "team_runner",
|
|
224
|
+
...(baseMetadata?.parentEventId ? { parentEventId: baseMetadata.parentEventId } : {}),
|
|
225
|
+
...(baseMetadata?.attemptId ? { attemptId: baseMetadata.attemptId } : {}),
|
|
226
|
+
...(baseMetadata?.branchId ? { branchId: baseMetadata.branchId } : {}),
|
|
227
|
+
...(baseMetadata?.causationId ? { causationId: baseMetadata.causationId } : {}),
|
|
228
|
+
...(baseMetadata?.correlationId ? { correlationId: baseMetadata.correlationId } : {}),
|
|
229
|
+
...(baseMetadata?.sessionIdentity ? { sessionIdentity: baseMetadata.sessionIdentity } : {}),
|
|
230
|
+
...(baseMetadata?.ownership ? { ownership: baseMetadata.ownership } : {}),
|
|
231
|
+
...(baseMetadata?.nudgeId ? { nudgeId: baseMetadata.nudgeId } : {}),
|
|
232
|
+
...(baseMetadata?.confidence ? { confidence: baseMetadata.confidence } : {}),
|
|
233
|
+
};
|
|
234
|
+
const fullEvent: TeamEvent = {
|
|
235
|
+
time: new Date().toISOString(),
|
|
236
|
+
...event,
|
|
237
|
+
metadata,
|
|
238
|
+
};
|
|
239
|
+
if (baseMetadata?.fingerprint || TERMINAL_EVENT_TYPES.has(fullEvent.type)) {
|
|
240
|
+
metadata = { ...metadata, fingerprint: baseMetadata?.fingerprint ?? computeEventFingerprint(fullEvent) };
|
|
241
|
+
fullEvent.metadata = metadata;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Overflow handling: same logic as sync path
|
|
245
|
+
const isTerminal = TERMINAL_EVENT_TYPES.has(fullEvent.type);
|
|
246
|
+
let skippedDueToSize = false;
|
|
247
|
+
if (!isTerminal && fs.existsSync(eventsPath)) {
|
|
248
|
+
const stat = fs.statSync(eventsPath);
|
|
249
|
+
if (stat.size > MAX_EVENTS_BYTES) {
|
|
250
|
+
try {
|
|
251
|
+
compactEventLog(eventsPath);
|
|
252
|
+
} catch (error) {
|
|
253
|
+
logInternalError("event-log.immediate-compact", error, `eventsPath=${eventsPath}`);
|
|
254
|
+
}
|
|
255
|
+
if (fs.existsSync(eventsPath)) {
|
|
256
|
+
const afterCompact = fs.statSync(eventsPath);
|
|
257
|
+
if (afterCompact.size > MAX_EVENTS_BYTES) {
|
|
258
|
+
rotateEventLog(eventsPath);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
try {
|
|
264
|
+
if (fs.existsSync(eventsPath) && fs.statSync(eventsPath).size > MAX_EVENTS_BYTES) {
|
|
265
|
+
logInternalError("event-log.size-limit", new Error(`events file ${eventsPath} exceeds ${MAX_EVENTS_BYTES} bytes after compaction`), `eventsPath=${eventsPath}`);
|
|
266
|
+
skippedDueToSize = true;
|
|
267
|
+
}
|
|
268
|
+
} catch (error) {
|
|
269
|
+
logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
if (!skippedDueToSize) {
|
|
273
|
+
const line = JSON.stringify(redactSecrets(fullEvent)) + "\n";
|
|
274
|
+
await fs.promises.appendFile(eventsPath, line, { encoding: "utf-8" });
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
appendCounter++;
|
|
278
|
+
if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
|
|
279
|
+
try { compactEventLog(eventsPath); } catch (error) { logInternalError("event-log.rotation", error, `eventsPath=${eventsPath}`); }
|
|
280
|
+
}
|
|
281
|
+
try { emitFromTeamEvent(fullEvent); } catch (error) { logInternalError("event-log.emit", error); }
|
|
282
|
+
|
|
283
|
+
const seq = fullEvent.metadata?.seq ?? 0;
|
|
284
|
+
try {
|
|
285
|
+
const stat = fs.statSync(eventsPath);
|
|
286
|
+
if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
|
|
287
|
+
evictOldestSequenceCacheEntries();
|
|
288
|
+
}
|
|
289
|
+
sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq });
|
|
290
|
+
persistSequence(eventsPath, seq);
|
|
291
|
+
} catch (error) {
|
|
292
|
+
logInternalError("event-log.persist-sequence", error, `eventsPath=${eventsPath}`);
|
|
293
|
+
}
|
|
294
|
+
return fullEvent;
|
|
295
|
+
});
|
|
296
|
+
asyncQueues.set(queueKey, next.catch(() => {}));
|
|
297
|
+
return next;
|
|
298
|
+
}
|
|
299
|
+
|
|
181
300
|
/**
|
|
182
301
|
* Body of `appendEvent` assuming the caller already holds
|
|
183
302
|
* `withEventLogLockSync` for `eventsPath`. Used by `appendEventBuffered` to
|
|
@@ -254,7 +373,7 @@ function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): Team
|
|
|
254
373
|
try {
|
|
255
374
|
const stat = fs.statSync(eventsPath);
|
|
256
375
|
if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
|
|
257
|
-
|
|
376
|
+
evictOldestSequenceCacheEntries();
|
|
258
377
|
}
|
|
259
378
|
sequenceCache.set(eventsPath, { size: stat.size, mtimeMs: stat.mtimeMs, seq });
|
|
260
379
|
persistSequence(eventsPath, seq);
|
|
@@ -283,6 +402,12 @@ const bufferedTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
|
283
402
|
const DEFAULT_BUFFER_MS = 20;
|
|
284
403
|
|
|
285
404
|
export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent, bufferMs = DEFAULT_BUFFER_MS): Promise<TeamEvent> {
|
|
405
|
+
// FIX: Terminal events must bypass buffer to ensure they're written immediately.
|
|
406
|
+
// Previously, terminal events like task.failed could be lost on process crash.
|
|
407
|
+
if (TERMINAL_EVENT_TYPES.has(event.type)) {
|
|
408
|
+
// For terminal events, write synchronously to ensure durability
|
|
409
|
+
return Promise.resolve(appendEvent(eventsPath, event));
|
|
410
|
+
}
|
|
286
411
|
return new Promise<TeamEvent>((resolve, reject) => {
|
|
287
412
|
const queue = bufferedQueues.get(eventsPath) ?? [];
|
|
288
413
|
queue.push({ event, resolve, reject });
|
|
@@ -325,12 +450,13 @@ export function flushEventLogBuffer(): void {
|
|
|
325
450
|
}
|
|
326
451
|
|
|
327
452
|
/**
|
|
328
|
-
*
|
|
329
|
-
* the
|
|
330
|
-
* (high-frequency `task.progress`).
|
|
453
|
+
* Schedule an async event append without waiting for the result.
|
|
454
|
+
* Uses the non-blocking async queue to avoid blocking the event loop.
|
|
455
|
+
* Use only for events whose return value is ignored (high-frequency `task.progress`).
|
|
456
|
+
* Errors are logged via logInternalError.
|
|
331
457
|
*/
|
|
332
|
-
export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEvent,
|
|
333
|
-
|
|
458
|
+
export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEvent, _bufferMs = DEFAULT_BUFFER_MS): void {
|
|
459
|
+
appendEventAsync(eventsPath, event).catch((error) => logInternalError("event-log.fire-and-forget", error, eventsPath));
|
|
334
460
|
}
|
|
335
461
|
|
|
336
462
|
// Auto-flush on process exit so buffered events do not silently leak.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hook-to-instinct bridge - connects crewHooks events to instinct formation.
|
|
3
|
+
* Auto-initializes when imported.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { crewHooks } from "../runtime/crew-hooks.ts";
|
|
7
|
+
|
|
8
|
+
// Lazy-initialized store and paths
|
|
9
|
+
let storeInstance: import("./instinct-store.js").InstinctStore | null = null;
|
|
10
|
+
let pathsInstance: typeof import("../utils/paths.js") | null = null;
|
|
11
|
+
|
|
12
|
+
async function getStore() {
|
|
13
|
+
if (!storeInstance) {
|
|
14
|
+
const { InstinctStore } = await import("./instinct-store.js");
|
|
15
|
+
const paths = await import("../utils/paths.js");
|
|
16
|
+
storeInstance = new InstinctStore(paths.projectCrewRoot(process.cwd()));
|
|
17
|
+
}
|
|
18
|
+
return storeInstance;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function getPaths() {
|
|
22
|
+
if (!pathsInstance) {
|
|
23
|
+
pathsInstance = await import("../utils/paths.js");
|
|
24
|
+
}
|
|
25
|
+
return pathsInstance;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Subscribe to events
|
|
29
|
+
crewHooks.register("task_completed", async (event) => {
|
|
30
|
+
try {
|
|
31
|
+
const store = await getStore();
|
|
32
|
+
if (event.data?.role) {
|
|
33
|
+
store.saveInstinct({
|
|
34
|
+
trigger: `role:${event.data.role}`,
|
|
35
|
+
action: "prefer",
|
|
36
|
+
confidence: 0.6,
|
|
37
|
+
scope: "global",
|
|
38
|
+
evidence: [`task:${event.taskId} completed`],
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
} catch {
|
|
42
|
+
// Best-effort - don't crash on instinct formation failures
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
crewHooks.register("task_failed", async (event) => {
|
|
47
|
+
try {
|
|
48
|
+
const store = await getStore();
|
|
49
|
+
if (event.data?.role) {
|
|
50
|
+
store.saveInstinct({
|
|
51
|
+
trigger: `role:${event.data.role}`,
|
|
52
|
+
action: "avoid",
|
|
53
|
+
confidence: 0.3,
|
|
54
|
+
scope: "global",
|
|
55
|
+
evidence: [`task:${event.taskId} failed`],
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
} catch {
|
|
59
|
+
// Best-effort
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
crewHooks.register("run_completed", async (event) => {
|
|
64
|
+
try {
|
|
65
|
+
const store = await getStore();
|
|
66
|
+
if (event.data?.taskCount) {
|
|
67
|
+
store.saveInstinct({
|
|
68
|
+
trigger: "run_completed",
|
|
69
|
+
action: `completed:${event.data.taskCount}tasks`,
|
|
70
|
+
confidence: 0.6,
|
|
71
|
+
scope: "global",
|
|
72
|
+
evidence: [`run:${event.runId}`],
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
} catch {
|
|
76
|
+
// Best-effort
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Get instinct-based recommendations.
|
|
82
|
+
*/
|
|
83
|
+
export async function getInstinctRecommendations() {
|
|
84
|
+
try {
|
|
85
|
+
const store = await getStore();
|
|
86
|
+
return store.getInstincts().filter((i: { confidence: number }) => i.confidence >= 0.6);
|
|
87
|
+
} catch {
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hook integrations - subscribes to crewHooks and provides observability.
|
|
3
|
+
* Auto-initializes when imported.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { crewHooks } from "../runtime/crew-hooks.ts";
|
|
7
|
+
|
|
8
|
+
// Statistics
|
|
9
|
+
let tasksCompleted = 0;
|
|
10
|
+
let tasksFailed = 0;
|
|
11
|
+
let runsCompleted = 0;
|
|
12
|
+
let runsFailed = 0;
|
|
13
|
+
|
|
14
|
+
// Subscribe to events (fire-and-forget)
|
|
15
|
+
crewHooks.register("task_completed", () => {
|
|
16
|
+
tasksCompleted++;
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
crewHooks.register("task_failed", () => {
|
|
20
|
+
tasksFailed++;
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
crewHooks.register("run_completed", () => {
|
|
24
|
+
runsCompleted++;
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
crewHooks.register("run_failed", () => {
|
|
28
|
+
runsFailed++;
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Get current hook statistics.
|
|
33
|
+
*/
|
|
34
|
+
export function getHookStats(): {
|
|
35
|
+
tasksCompleted: number;
|
|
36
|
+
tasksFailed: number;
|
|
37
|
+
runsCompleted: number;
|
|
38
|
+
runsFailed: number;
|
|
39
|
+
} {
|
|
40
|
+
return { tasksCompleted, tasksFailed, runsCompleted, runsFailed };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Reset statistics (useful for testing).
|
|
45
|
+
*/
|
|
46
|
+
export function resetHookStats(): void {
|
|
47
|
+
tasksCompleted = 0;
|
|
48
|
+
tasksFailed = 0;
|
|
49
|
+
runsCompleted = 0;
|
|
50
|
+
runsFailed = 0;
|
|
51
|
+
}
|