pi-crew 0.6.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +225 -0
- package/README.md +70 -31
- package/docs/issue-29-analysis.md +189 -0
- package/docs/superpowers/plans/2026-06-09-fallow-patterns-adoption.md +1268 -0
- package/package.json +2 -2
- package/src/agents/agent-config.ts +2 -1
- package/src/benchmark/feedback-loop.ts +4 -2
- package/src/config/config.ts +106 -15
- package/src/errors.ts +107 -0
- package/src/extension/async-notifier.ts +6 -2
- package/src/extension/crew-cleanup.ts +8 -5
- package/src/extension/cross-extension-rpc.ts +48 -0
- package/src/extension/management.ts +464 -109
- package/src/extension/register.ts +194 -34
- package/src/extension/registration/commands.ts +4 -3
- package/src/extension/registration/subagent-helpers.ts +2 -2
- package/src/extension/registration/subagent-tools.ts +3 -1
- package/src/extension/registration/team-tool.ts +3 -1
- package/src/extension/registration/viewers.ts +3 -2
- package/src/extension/run-export.ts +16 -1
- package/src/extension/run-import.ts +16 -0
- package/src/extension/team-tool/anchor.ts +5 -1
- package/src/extension/team-tool/api.ts +12 -7
- package/src/extension/team-tool/cancel.ts +3 -3
- package/src/extension/team-tool/config-patch.ts +15 -1
- package/src/extension/team-tool/explain.ts +1 -1
- package/src/extension/team-tool/handle-schedule.ts +40 -0
- package/src/extension/team-tool/inspect.ts +3 -3
- package/src/extension/team-tool/lifecycle-actions.ts +4 -4
- package/src/extension/team-tool/respond.ts +2 -2
- package/src/extension/team-tool/run.ts +60 -11
- package/src/extension/team-tool/status.ts +1 -1
- package/src/extension/team-tool.ts +175 -47
- package/src/hooks/registry.ts +84 -12
- package/src/hooks/types.ts +12 -3
- package/src/i18n.ts +15 -2
- package/src/observability/exporters/otlp-exporter.ts +73 -0
- package/src/plugins/plugin-define.ts +6 -0
- package/src/plugins/plugin-registry.ts +32 -0
- package/src/plugins/plugins/index.ts +3 -0
- package/src/plugins/plugins/nextjs.ts +19 -0
- package/src/plugins/plugins/vite.ts +14 -0
- package/src/plugins/plugins/vitest.ts +14 -0
- package/src/runtime/adaptive-plan.ts +24 -0
- package/src/runtime/agent-control.ts +6 -3
- package/src/runtime/async-runner.ts +86 -3
- package/src/runtime/background-runner.ts +529 -144
- package/src/runtime/chain-parser.ts +5 -1
- package/src/runtime/chain-runner.ts +67 -3
- package/src/runtime/checkpoint.ts +262 -180
- package/src/runtime/child-pi.ts +165 -38
- package/src/runtime/crash-recovery.ts +25 -14
- package/src/runtime/crew-agent-records.ts +6 -3
- package/src/runtime/cross-extension-rpc.ts +34 -8
- package/src/runtime/diagnostic-export.ts +4 -5
- package/src/runtime/dynamic-script-runner.ts +9 -6
- package/src/runtime/foreground-watchdog.ts +3 -3
- package/src/runtime/heartbeat-watcher.ts +19 -2
- package/src/runtime/intercom-bridge.ts +7 -0
- package/src/runtime/iteration-hooks.ts +1 -1
- package/src/runtime/live-agent-manager.ts +6 -3
- package/src/runtime/live-irc.ts +4 -2
- package/src/runtime/manifest-cache.ts +4 -0
- package/src/runtime/orphan-worker-registry.ts +444 -0
- package/src/runtime/parallel-utils.ts +2 -1
- package/src/runtime/parent-guard.ts +70 -16
- package/src/runtime/pi-args.ts +437 -14
- package/src/runtime/pi-spawn.ts +1 -0
- package/src/runtime/post-checks.ts +11 -4
- package/src/runtime/retry-runner.ts +9 -3
- package/src/runtime/{drift-detectors.ts → run-drift.ts} +1 -1
- package/src/runtime/run-tracker.ts +38 -10
- package/src/runtime/sandbox.ts +42 -21
- package/src/runtime/scheduler.ts +25 -2
- package/src/runtime/semaphore.ts +2 -1
- package/src/runtime/settings-store.ts +14 -2
- package/src/runtime/skill-effectiveness.ts +109 -64
- package/src/runtime/skill-instructions.ts +114 -33
- package/src/runtime/stale-reconciler.ts +310 -69
- package/src/runtime/subagent-manager.ts +265 -53
- package/src/runtime/subprocess-tool-registry.ts +2 -2
- package/src/runtime/task-health.ts +76 -0
- package/src/runtime/task-id.ts +20 -0
- package/src/runtime/task-packet.ts +13 -1
- package/src/runtime/task-runner/live-executor.ts +1 -1
- package/src/runtime/task-runner/progress.ts +1 -1
- package/src/runtime/task-runner/state-helpers.ts +110 -6
- package/src/runtime/task-runner.ts +98 -67
- package/src/runtime/team-runner.ts +186 -20
- package/src/runtime/usage-tracker.ts +4 -2
- package/src/runtime/verification-gates.ts +36 -9
- package/src/schema/team-tool-schema.ts +27 -9
- package/src/skills/discover-skills.ts +9 -3
- package/src/state/active-run-registry.ts +170 -38
- package/src/state/artifact-store.ts +25 -13
- package/src/state/atomic-write-v2.ts +86 -0
- package/src/state/atomic-write.ts +346 -55
- package/src/state/blob-store.ts +178 -10
- package/src/state/contracts.ts +2 -1
- package/src/state/crew-init.ts +161 -28
- package/src/state/decision-ledger.ts +172 -111
- package/src/state/event-log-rotation.ts +82 -52
- package/src/state/event-log.ts +270 -75
- package/src/state/health-store.ts +71 -0
- package/src/state/hook-instinct-bridge.ts +2 -1
- package/src/state/locks.ts +109 -20
- package/src/state/mailbox.ts +45 -7
- package/src/state/observation-store.ts +4 -1
- package/src/state/run-graph.ts +141 -130
- package/src/state/run-metrics.ts +24 -8
- package/src/state/state-store.ts +333 -44
- package/src/state/task-claims.ts +9 -2
- package/src/tools/safe-bash.ts +69 -20
- package/src/types/new-api-types.ts +10 -5
- package/src/ui/keybinding-map.ts +2 -1
- package/src/ui/live-run-sidebar.ts +1 -1
- package/src/ui/loaders.ts +4 -0
- package/src/ui/overlays/agent-picker-overlay.ts +1 -1
- package/src/ui/overlays/mailbox-detail-overlay.ts +1 -1
- package/src/ui/run-action-dispatcher.ts +4 -3
- package/src/ui/run-snapshot-cache.ts +1 -1
- package/src/ui/status-colors.ts +2 -1
- package/src/ui/syntax-highlight.ts +2 -1
- package/src/ui/tool-render.ts +13 -3
- package/src/utils/env-filter.ts +66 -0
- package/src/utils/file-coalescer.ts +9 -1
- package/src/utils/fs-watch.ts +4 -2
- package/src/utils/gh-protocol.ts +2 -1
- package/src/utils/paths.ts +80 -5
- package/src/utils/redaction.ts +6 -1
- package/src/utils/safe-paths.ts +287 -10
- package/src/worktree/cleanup.ts +117 -26
- package/src/worktree/worktree-manager.ts +128 -15
- package/test-bugs-all.mjs +10 -6
- package/test-integration-check.ts +4 -0
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Skill Effectiveness — ECC INSTINCT/CONFIDENCE Pattern Implementation
|
|
3
|
-
*
|
|
3
|
+
*
|
|
4
4
|
* Implements confidence-weighted skill activation based on ECC's instinct system.
|
|
5
5
|
* Tracks skill activation success and adjusts confidence scores.
|
|
6
|
-
*
|
|
6
|
+
*
|
|
7
7
|
* Based on: docs/distillation/ECC-hooks-instincts.md §2-3 (instinct system, confidence thresholds)
|
|
8
8
|
* Based on: docs/distillation/ECC-10-skills.md §8 (continuous-learning-v2)
|
|
9
|
-
*
|
|
9
|
+
*
|
|
10
10
|
* @module skill-effectiveness
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
14
14
|
import { dirname, join } from "path";
|
|
15
|
+
import { projectCrewRoot } from "../utils/paths.ts";
|
|
15
16
|
import { crewHooks } from "./crew-hooks.ts";
|
|
16
17
|
|
|
17
18
|
/**
|
|
@@ -19,10 +20,10 @@ import { crewHooks } from "./crew-hooks.ts";
|
|
|
19
20
|
* Skills below 0.3 threshold are considered tentative and not enforced.
|
|
20
21
|
*/
|
|
21
22
|
export const CONFIDENCE_THRESHOLDS = {
|
|
22
|
-
TENTATIVE: 0.3,
|
|
23
|
-
MODERATE: 0.5,
|
|
24
|
-
STRONG: 0.7,
|
|
25
|
-
NEAR_CERTAIN: 0.9,
|
|
23
|
+
TENTATIVE: 0.3, // Suggested but not enforced
|
|
24
|
+
MODERATE: 0.5, // Applied when relevant
|
|
25
|
+
STRONG: 0.7, // Auto-approved for application
|
|
26
|
+
NEAR_CERTAIN: 0.9, // Core behavior
|
|
26
27
|
} as const;
|
|
27
28
|
|
|
28
29
|
/**
|
|
@@ -30,12 +31,12 @@ export const CONFIDENCE_THRESHOLDS = {
|
|
|
30
31
|
* From ECC instinct system: 1-2 observations → 0.3, 3-5 → 0.5, etc.
|
|
31
32
|
*/
|
|
32
33
|
export const INITIAL_CONFIDENCE_BY_FREQUENCY: Record<string, number> = {
|
|
33
|
-
"1": 0.3,
|
|
34
|
-
"2": 0.3,
|
|
35
|
-
"3": 0.5,
|
|
34
|
+
"1": 0.3, // 1 observation → tentative
|
|
35
|
+
"2": 0.3, // 2 observations → tentative
|
|
36
|
+
"3": 0.5, // 3 observations → moderate
|
|
36
37
|
"4": 0.5,
|
|
37
38
|
"5": 0.5,
|
|
38
|
-
"6": 0.7,
|
|
39
|
+
"6": 0.7, // 6-10 observations → strong
|
|
39
40
|
"7": 0.7,
|
|
40
41
|
"8": 0.7,
|
|
41
42
|
"9": 0.7,
|
|
@@ -47,8 +48,8 @@ export const INITIAL_CONFIDENCE_BY_FREQUENCY: Record<string, number> = {
|
|
|
47
48
|
* Confidence adjustments per ECC instinct system.
|
|
48
49
|
*/
|
|
49
50
|
export const CONFIDENCE_ADJUSTMENTS = {
|
|
50
|
-
CONFIRMING: 0.05,
|
|
51
|
-
CONTRADICTING: -0.1,
|
|
51
|
+
CONFIRMING: 0.05, // Each confirming observation
|
|
52
|
+
CONTRADICTING: -0.1, // Each contradicting observation
|
|
52
53
|
DECAY_PER_WEEK: -0.02, // Per week without observation
|
|
53
54
|
} as const;
|
|
54
55
|
|
|
@@ -57,24 +58,24 @@ export const CONFIDENCE_ADJUSTMENTS = {
|
|
|
57
58
|
* Skill can be promoted to "strong enforcement" when these are met.
|
|
58
59
|
*/
|
|
59
60
|
export const PROMOTION_GATE_CRITERIA = {
|
|
60
|
-
MIN_CORRECTNESS: 0.8,
|
|
61
|
-
MIN_ACTIVATIONS: 5,
|
|
62
|
-
MIN_AVG_CONFIDENCE: 0.7,
|
|
61
|
+
MIN_CORRECTNESS: 0.8, // 80% pass rate
|
|
62
|
+
MIN_ACTIVATIONS: 5, // Minimum observations before filtering
|
|
63
|
+
MIN_AVG_CONFIDENCE: 0.7, // Average confidence threshold
|
|
63
64
|
} as const;
|
|
64
65
|
|
|
65
66
|
/**
|
|
66
67
|
* Skill activation record - captures each time a skill is used.
|
|
67
68
|
*/
|
|
68
69
|
export interface SkillActivation {
|
|
69
|
-
id: string;
|
|
70
|
-
skillId: string;
|
|
71
|
-
role: string;
|
|
72
|
-
runId: string;
|
|
73
|
-
taskId: string;
|
|
74
|
-
timestamp: string;
|
|
75
|
-
passed: boolean;
|
|
76
|
-
outcome?: string;
|
|
77
|
-
confidence: number;
|
|
70
|
+
id: string; // Unique activation ID
|
|
71
|
+
skillId: string; // Skill identifier (e.g., "verification-before-done")
|
|
72
|
+
role: string; // Role that activated the skill
|
|
73
|
+
runId: string; // Run ID
|
|
74
|
+
taskId: string; // Task ID
|
|
75
|
+
timestamp: string; // ISO timestamp
|
|
76
|
+
passed: boolean; // Whether the skill was successfully applied
|
|
77
|
+
outcome?: string; // Optional outcome description
|
|
78
|
+
confidence: number; // Confidence at time of activation
|
|
78
79
|
}
|
|
79
80
|
|
|
80
81
|
/**
|
|
@@ -85,13 +86,13 @@ export interface SkillMetrics {
|
|
|
85
86
|
totalActivations: number;
|
|
86
87
|
passedActivations: number;
|
|
87
88
|
failedActivations: number;
|
|
88
|
-
passRate: number;
|
|
89
|
-
avgConfidence: number;
|
|
90
|
-
currentConfidence: number;
|
|
89
|
+
passRate: number; // passed / total
|
|
90
|
+
avgConfidence: number; // Rolling average confidence
|
|
91
|
+
currentConfidence: number; // Current confidence score
|
|
91
92
|
trend: "improving" | "stable" | "declining";
|
|
92
|
-
lastActivation?: string;
|
|
93
|
-
firstActivation?: string;
|
|
94
|
-
roleBreakdown: Record<string, number>;
|
|
93
|
+
lastActivation?: string; // ISO timestamp
|
|
94
|
+
firstActivation?: string; // ISO timestamp
|
|
95
|
+
roleBreakdown: Record<string, number>; // Activations per role
|
|
95
96
|
}
|
|
96
97
|
|
|
97
98
|
/**
|
|
@@ -102,35 +103,39 @@ export interface WeightedSkill {
|
|
|
102
103
|
confidence: number;
|
|
103
104
|
threshold: keyof typeof CONFIDENCE_THRESHOLDS;
|
|
104
105
|
behavior: "suggest" | "apply_if_asked" | "apply_auto" | "act_autonomous";
|
|
105
|
-
evidence: string;
|
|
106
|
+
evidence: string; // Evidence for confidence score
|
|
106
107
|
metrics: SkillMetrics;
|
|
107
108
|
}
|
|
108
109
|
|
|
109
110
|
/**
|
|
110
111
|
* Get skill effectiveness storage path.
|
|
112
|
+
* Uses projectCrewRoot() so the path honours the `.pi/teams/` fallback
|
|
113
|
+
* for `.pi`-based projects (see issue #29).
|
|
111
114
|
*/
|
|
112
|
-
function getSkillMetricsPath(runId: string): string {
|
|
115
|
+
function getSkillMetricsPath(cwd: string, runId: string): string {
|
|
113
116
|
return join(
|
|
114
|
-
|
|
115
|
-
|
|
117
|
+
projectCrewRoot(cwd),
|
|
118
|
+
`state/runs/${runId}/skill-metrics.jsonl`,
|
|
116
119
|
);
|
|
117
120
|
}
|
|
118
121
|
|
|
119
122
|
/**
|
|
120
123
|
* Get skill activations path.
|
|
124
|
+
* Uses projectCrewRoot() so the path honours the `.pi/teams/` fallback
|
|
125
|
+
* for `.pi`-based projects (see issue #29).
|
|
121
126
|
*/
|
|
122
|
-
function getSkillActivationsPath(runId: string): string {
|
|
127
|
+
function getSkillActivationsPath(cwd: string, runId: string): string {
|
|
123
128
|
return join(
|
|
124
|
-
|
|
125
|
-
|
|
129
|
+
projectCrewRoot(cwd),
|
|
130
|
+
`state/runs/${runId}/skill-activations.jsonl`,
|
|
126
131
|
);
|
|
127
132
|
}
|
|
128
133
|
|
|
129
134
|
/**
|
|
130
135
|
* Ensure directory exists for skill metrics.
|
|
131
136
|
*/
|
|
132
|
-
function ensureSkillMetricsDir(runId: string): void {
|
|
133
|
-
const dir = dirname(getSkillMetricsPath(runId));
|
|
137
|
+
function ensureSkillMetricsDir(cwd: string, runId: string): void {
|
|
138
|
+
const dir = dirname(getSkillMetricsPath(cwd, runId));
|
|
134
139
|
if (!existsSync(dir)) {
|
|
135
140
|
mkdirSync(dir, { recursive: true });
|
|
136
141
|
}
|
|
@@ -163,7 +168,9 @@ export function adjustConfidence(current: number, passed: boolean): number {
|
|
|
163
168
|
export function applyDecay(current: number, lastActivation?: string): number {
|
|
164
169
|
if (!lastActivation) return current;
|
|
165
170
|
|
|
166
|
-
const daysSince =
|
|
171
|
+
const daysSince =
|
|
172
|
+
(Date.now() - new Date(lastActivation).getTime()) /
|
|
173
|
+
(1000 * 60 * 60 * 24);
|
|
167
174
|
const decayWeeks = Math.floor(daysSince / 7);
|
|
168
175
|
const decay = decayWeeks * CONFIDENCE_ADJUSTMENTS.DECAY_PER_WEEK;
|
|
169
176
|
|
|
@@ -173,8 +180,11 @@ export function applyDecay(current: number, lastActivation?: string): number {
|
|
|
173
180
|
/**
|
|
174
181
|
* Determine behavior based on confidence threshold.
|
|
175
182
|
*/
|
|
176
|
-
export function confidenceToBehavior(
|
|
177
|
-
|
|
183
|
+
export function confidenceToBehavior(
|
|
184
|
+
confidence: number,
|
|
185
|
+
): WeightedSkill["behavior"] {
|
|
186
|
+
if (confidence >= CONFIDENCE_THRESHOLDS.NEAR_CERTAIN)
|
|
187
|
+
return "act_autonomous";
|
|
178
188
|
if (confidence >= CONFIDENCE_THRESHOLDS.STRONG) return "apply_auto";
|
|
179
189
|
if (confidence >= CONFIDENCE_THRESHOLDS.MODERATE) return "apply_if_asked";
|
|
180
190
|
return "suggest";
|
|
@@ -183,7 +193,9 @@ export function confidenceToBehavior(confidence: number): WeightedSkill["behavio
|
|
|
183
193
|
/**
|
|
184
194
|
* Determine threshold name from confidence.
|
|
185
195
|
*/
|
|
186
|
-
export function confidenceToThreshold(
|
|
196
|
+
export function confidenceToThreshold(
|
|
197
|
+
confidence: number,
|
|
198
|
+
): keyof typeof CONFIDENCE_THRESHOLDS {
|
|
187
199
|
if (confidence >= CONFIDENCE_THRESHOLDS.NEAR_CERTAIN) return "NEAR_CERTAIN";
|
|
188
200
|
if (confidence >= CONFIDENCE_THRESHOLDS.STRONG) return "STRONG";
|
|
189
201
|
if (confidence >= CONFIDENCE_THRESHOLDS.TENTATIVE) return "MODERATE";
|
|
@@ -195,11 +207,12 @@ export function confidenceToThreshold(confidence: number): keyof typeof CONFIDEN
|
|
|
195
207
|
* Appends to the run's skill-activations.jsonl for learning.
|
|
196
208
|
*/
|
|
197
209
|
export function recordSkillActivation(
|
|
210
|
+
cwd: string,
|
|
198
211
|
activation: SkillActivation,
|
|
199
212
|
): SkillActivation {
|
|
200
|
-
ensureSkillMetricsDir(activation.runId);
|
|
213
|
+
ensureSkillMetricsDir(cwd, activation.runId);
|
|
201
214
|
|
|
202
|
-
const path = getSkillActivationsPath(activation.runId);
|
|
215
|
+
const path = getSkillActivationsPath(cwd, activation.runId);
|
|
203
216
|
const line = JSON.stringify(activation) + "\n";
|
|
204
217
|
writeFileSync(path, line, { flag: "a", encoding: "utf-8" });
|
|
205
218
|
|
|
@@ -209,8 +222,11 @@ export function recordSkillActivation(
|
|
|
209
222
|
/**
|
|
210
223
|
* Get all skill activations for a run.
|
|
211
224
|
*/
|
|
212
|
-
export function getSkillActivations(
|
|
213
|
-
|
|
225
|
+
export function getSkillActivations(
|
|
226
|
+
cwd: string,
|
|
227
|
+
runId: string,
|
|
228
|
+
): SkillActivation[] {
|
|
229
|
+
const path = getSkillActivationsPath(cwd, runId);
|
|
214
230
|
|
|
215
231
|
if (!existsSync(path)) {
|
|
216
232
|
return [];
|
|
@@ -256,11 +272,13 @@ export function computeSkillMetrics(
|
|
|
256
272
|
skillActivations.reduce((sum, a) => sum + a.confidence, 0) /
|
|
257
273
|
skillActivations.length;
|
|
258
274
|
const currentConfidence =
|
|
259
|
-
skillActivations[skillActivations.length - 1]?.confidence ??
|
|
275
|
+
skillActivations[skillActivations.length - 1]?.confidence ??
|
|
276
|
+
avgConfidence;
|
|
260
277
|
|
|
261
278
|
// Compute trend from last 5 activations
|
|
262
279
|
const recent = skillActivations.slice(-5);
|
|
263
|
-
const recentPassRate =
|
|
280
|
+
const recentPassRate =
|
|
281
|
+
recent.filter((a) => a.passed).length / recent.length;
|
|
264
282
|
const earlier = skillActivations.slice(0, -5);
|
|
265
283
|
const earlierPassRate =
|
|
266
284
|
earlier.length > 0
|
|
@@ -282,7 +300,8 @@ export function computeSkillMetrics(
|
|
|
282
300
|
}
|
|
283
301
|
|
|
284
302
|
// Apply decay if not observed recently
|
|
285
|
-
const lastActivation =
|
|
303
|
+
const lastActivation =
|
|
304
|
+
skillActivations[skillActivations.length - 1]?.timestamp;
|
|
286
305
|
const decayedConfidence = applyDecay(currentConfidence, lastActivation);
|
|
287
306
|
|
|
288
307
|
return {
|
|
@@ -315,10 +334,13 @@ export function evaluatePromotionGate(metrics: SkillMetrics): {
|
|
|
315
334
|
reason: string;
|
|
316
335
|
} {
|
|
317
336
|
const criteria = {
|
|
318
|
-
correctness:
|
|
319
|
-
|
|
337
|
+
correctness:
|
|
338
|
+
metrics.passRate >= PROMOTION_GATE_CRITERIA.MIN_CORRECTNESS,
|
|
339
|
+
evidence:
|
|
340
|
+
metrics.totalActivations >= PROMOTION_GATE_CRITERIA.MIN_ACTIVATIONS,
|
|
320
341
|
rollback: metrics.trend !== "declining",
|
|
321
|
-
encoding:
|
|
342
|
+
encoding:
|
|
343
|
+
metrics.avgConfidence >= PROMOTION_GATE_CRITERIA.MIN_AVG_CONFIDENCE,
|
|
322
344
|
};
|
|
323
345
|
|
|
324
346
|
const allPassed = Object.values(criteria).every(Boolean);
|
|
@@ -341,12 +363,13 @@ export function evaluatePromotionGate(metrics: SkillMetrics): {
|
|
|
341
363
|
* Filters by minimum confidence threshold.
|
|
342
364
|
*/
|
|
343
365
|
export function getWeightedSkillsForRole(
|
|
366
|
+
cwd: string,
|
|
344
367
|
role: string,
|
|
345
368
|
skillIds: string[],
|
|
346
369
|
runId: string,
|
|
347
370
|
minConfidence: number = CONFIDENCE_THRESHOLDS.TENTATIVE,
|
|
348
371
|
): WeightedSkill[] {
|
|
349
|
-
const activations = getSkillActivations(runId);
|
|
372
|
+
const activations = getSkillActivations(cwd, runId);
|
|
350
373
|
|
|
351
374
|
return skillIds
|
|
352
375
|
.map((skillId) => {
|
|
@@ -374,13 +397,21 @@ export function getWeightedSkillsForRole(
|
|
|
374
397
|
* Filter skills by confidence threshold.
|
|
375
398
|
* Skills below threshold are marked as "suggest" only.
|
|
376
399
|
*/
|
|
377
|
-
|
|
400
|
+
/** @internal */
|
|
401
|
+
function filterSkillsByConfidence(
|
|
402
|
+
cwd: string,
|
|
378
403
|
skillIds: string[],
|
|
379
404
|
runId: string,
|
|
380
405
|
threshold: keyof typeof CONFIDENCE_THRESHOLDS = "MODERATE",
|
|
381
406
|
): WeightedSkill[] {
|
|
382
407
|
const minConfidence = CONFIDENCE_THRESHOLDS[threshold];
|
|
383
|
-
return getWeightedSkillsForRole(
|
|
408
|
+
return getWeightedSkillsForRole(
|
|
409
|
+
cwd,
|
|
410
|
+
"global",
|
|
411
|
+
skillIds,
|
|
412
|
+
runId,
|
|
413
|
+
minConfidence,
|
|
414
|
+
);
|
|
384
415
|
}
|
|
385
416
|
|
|
386
417
|
/**
|
|
@@ -414,7 +445,11 @@ export function registerSkillEffectivenessHooks(): void {
|
|
|
414
445
|
passed: success,
|
|
415
446
|
confidence: computeInitialConfidence(1),
|
|
416
447
|
};
|
|
417
|
-
|
|
448
|
+
// cwd comes from the event payload (set by callers) so that the
|
|
449
|
+
// activation lands in the correct .pi/teams/ or .crew/state/runs/
|
|
450
|
+
// (see issue #29).
|
|
451
|
+
const eventCwd = (data?.cwd as string) ?? process.cwd();
|
|
452
|
+
recordSkillActivation(eventCwd, activation);
|
|
418
453
|
}
|
|
419
454
|
});
|
|
420
455
|
|
|
@@ -431,11 +466,13 @@ export function registerSkillEffectivenessHooks(): void {
|
|
|
431
466
|
/**
|
|
432
467
|
* Generate a skill effectiveness report for a run.
|
|
433
468
|
*/
|
|
434
|
-
|
|
469
|
+
/** @internal */
|
|
470
|
+
function generateSkillEffectivenessReport(
|
|
471
|
+
cwd: string,
|
|
435
472
|
runId: string,
|
|
436
473
|
skillIds: string[],
|
|
437
474
|
): string {
|
|
438
|
-
const activations = getSkillActivations(runId);
|
|
475
|
+
const activations = getSkillActivations(cwd, runId);
|
|
439
476
|
const lines: string[] = [
|
|
440
477
|
`# Skill Effectiveness Report: ${runId}`,
|
|
441
478
|
"",
|
|
@@ -457,13 +494,21 @@ export function generateSkillEffectivenessReport(
|
|
|
457
494
|
const gate = evaluatePromotionGate(metrics);
|
|
458
495
|
|
|
459
496
|
lines.push(`### ${skillId}`);
|
|
460
|
-
lines.push(
|
|
461
|
-
|
|
497
|
+
lines.push(
|
|
498
|
+
`- **Confidence**: ${metrics.currentConfidence.toFixed(2)} (${metrics.trend})`,
|
|
499
|
+
);
|
|
500
|
+
lines.push(
|
|
501
|
+
`- **Pass Rate**: ${(metrics.passRate * 100).toFixed(1)}% (${metrics.passedActivations}/${metrics.totalActivations})`,
|
|
502
|
+
);
|
|
462
503
|
lines.push(`- **Avg Confidence**: ${metrics.avgConfidence.toFixed(2)}`);
|
|
463
|
-
lines.push(
|
|
504
|
+
lines.push(
|
|
505
|
+
`- **Promotion Gate**: ${gate.passed ? "PASSED ✅" : "NOT MET"}`,
|
|
506
|
+
);
|
|
464
507
|
|
|
465
508
|
if (Object.keys(metrics.roleBreakdown).length > 0) {
|
|
466
|
-
lines.push(
|
|
509
|
+
lines.push(
|
|
510
|
+
`- **By Role**: ${JSON.stringify(metrics.roleBreakdown)}`,
|
|
511
|
+
);
|
|
467
512
|
}
|
|
468
513
|
|
|
469
514
|
lines.push("");
|
|
@@ -3,15 +3,24 @@ import * as path from "node:path";
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
5
5
|
import type { TeamRole } from "../teams/team-config.ts";
|
|
6
|
+
import {
|
|
7
|
+
isSafePathId,
|
|
8
|
+
resolveContainedPath,
|
|
9
|
+
resolveRealContainedPath,
|
|
10
|
+
} from "../utils/safe-paths.ts";
|
|
6
11
|
import type { WorkflowStep } from "../workflows/workflow-config.ts";
|
|
7
|
-
import { isSafePathId, resolveContainedPath, resolveRealContainedPath } from "../utils/safe-paths.ts";
|
|
8
12
|
import {
|
|
13
|
+
CONFIDENCE_THRESHOLDS,
|
|
9
14
|
getWeightedSkillsForRole,
|
|
10
15
|
registerSkillEffectivenessHooks,
|
|
11
|
-
CONFIDENCE_THRESHOLDS,
|
|
12
16
|
} from "./skill-effectiveness.ts";
|
|
13
17
|
|
|
14
|
-
const PACKAGE_SKILLS_DIR = path.resolve(
|
|
18
|
+
const PACKAGE_SKILLS_DIR = path.resolve(
|
|
19
|
+
path.dirname(fileURLToPath(import.meta.url)),
|
|
20
|
+
"..",
|
|
21
|
+
"..",
|
|
22
|
+
"skills",
|
|
23
|
+
);
|
|
15
24
|
const MAX_SKILL_CHARS = 1500;
|
|
16
25
|
const MAX_TOTAL_CHARS = 6000;
|
|
17
26
|
const MAX_SKILL_NAME_CHARS = 80;
|
|
@@ -23,7 +32,11 @@ const DEFAULT_ROLE_SKILLS: Record<string, string[]> = {
|
|
|
23
32
|
analyst: ["read-only-explorer", "requirements-to-task-packet"],
|
|
24
33
|
planner: ["delegation-patterns", "requirements-to-task-packet"],
|
|
25
34
|
critic: ["read-only-explorer", "multi-perspective-review"],
|
|
26
|
-
executor: [
|
|
35
|
+
executor: [
|
|
36
|
+
"state-mutation-locking",
|
|
37
|
+
"safe-bash",
|
|
38
|
+
"verification-before-done",
|
|
39
|
+
],
|
|
27
40
|
reviewer: ["read-only-explorer", "multi-perspective-review"],
|
|
28
41
|
// SECURITY NOTE: The following skill names are trusted package-level skills.
|
|
29
42
|
// If a project has a skills/ directory containing subdirectories with these names,
|
|
@@ -31,7 +44,10 @@ const DEFAULT_ROLE_SKILLS: Record<string, string[]> = {
|
|
|
31
44
|
// project dir before package dir) and their content injected verbatim into prompts.
|
|
32
45
|
// The "Applicable Skills" block will add an untrusted-content warning for project skills,
|
|
33
46
|
// but be aware this is a potential supply-chain risk in multi-contributor projects.
|
|
34
|
-
"security-reviewer": [
|
|
47
|
+
"security-reviewer": [
|
|
48
|
+
"secure-agent-orchestration-review",
|
|
49
|
+
"ownership-session-security",
|
|
50
|
+
],
|
|
35
51
|
"test-engineer": ["verification-before-done", "safe-bash"],
|
|
36
52
|
verifier: ["verification-before-done", "runtime-state-reader"],
|
|
37
53
|
writer: ["context-artifact-hygiene", "verify-evidence"],
|
|
@@ -50,11 +66,18 @@ export interface RenderSkillInstructionsInput extends ResolveTaskSkillsInput {
|
|
|
50
66
|
}
|
|
51
67
|
|
|
52
68
|
function isValidSkillName(name: string): boolean {
|
|
53
|
-
return
|
|
69
|
+
return (
|
|
70
|
+
name.length > 0 &&
|
|
71
|
+
name.length <= MAX_SKILL_NAME_CHARS &&
|
|
72
|
+
isSafePathId(name)
|
|
73
|
+
);
|
|
54
74
|
}
|
|
55
75
|
|
|
56
76
|
function sanitizeSkillName(name: string): string {
|
|
57
|
-
return
|
|
77
|
+
return (
|
|
78
|
+
name.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, MAX_SKILL_NAME_CHARS) ||
|
|
79
|
+
"invalid"
|
|
80
|
+
);
|
|
58
81
|
}
|
|
59
82
|
|
|
60
83
|
function unique(items: string[]): string[] {
|
|
@@ -69,11 +92,18 @@ function unique(items: string[]): string[] {
|
|
|
69
92
|
return result;
|
|
70
93
|
}
|
|
71
94
|
|
|
72
|
-
export function normalizeSkillOverride(
|
|
95
|
+
export function normalizeSkillOverride(
|
|
96
|
+
value: string | string[] | boolean | undefined,
|
|
97
|
+
): string[] | false | undefined {
|
|
73
98
|
if (value === false) return false;
|
|
74
|
-
if (typeof value === "string")
|
|
99
|
+
if (typeof value === "string")
|
|
100
|
+
return value
|
|
101
|
+
.split(",")
|
|
102
|
+
.map((entry) => entry.trim())
|
|
103
|
+
.filter(Boolean);
|
|
75
104
|
if (value === true) return undefined;
|
|
76
|
-
if (Array.isArray(value))
|
|
105
|
+
if (Array.isArray(value))
|
|
106
|
+
return value.map((entry) => entry.trim()).filter(Boolean);
|
|
77
107
|
return undefined;
|
|
78
108
|
}
|
|
79
109
|
|
|
@@ -81,13 +111,17 @@ export function defaultSkillsForRole(role: string): string[] {
|
|
|
81
111
|
return DEFAULT_ROLE_SKILLS[role] ?? [];
|
|
82
112
|
}
|
|
83
113
|
|
|
84
|
-
function collectTaskSkillNames(
|
|
114
|
+
function collectTaskSkillNames(
|
|
115
|
+
input: ResolveTaskSkillsInput | undefined,
|
|
116
|
+
): string[] {
|
|
85
117
|
if (!input) return [];
|
|
86
118
|
if (input.override === false) return [];
|
|
87
|
-
const roleDefaultsDisabled =
|
|
119
|
+
const roleDefaultsDisabled =
|
|
120
|
+
input.teamRole?.skills === false || input.step?.skills === false;
|
|
88
121
|
const names = roleDefaultsDisabled ? [] : defaultSkillsForRole(input.role);
|
|
89
122
|
if (input.agent?.skills?.length) names.push(...input.agent.skills);
|
|
90
|
-
if (Array.isArray(input.teamRole?.skills))
|
|
123
|
+
if (Array.isArray(input.teamRole?.skills))
|
|
124
|
+
names.push(...input.teamRole.skills);
|
|
91
125
|
if (Array.isArray(input.step?.skills)) names.push(...input.step.skills);
|
|
92
126
|
if (Array.isArray(input.override)) names.push(...input.override);
|
|
93
127
|
return unique(names);
|
|
@@ -103,10 +137,12 @@ export function resolveTaskSkillNames(input: ResolveTaskSkillsInput): string[] {
|
|
|
103
137
|
// See: SECURITY-ISSUES.md SEC-003
|
|
104
138
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
105
139
|
|
|
106
|
-
function candidateSkillDirs(
|
|
140
|
+
function candidateSkillDirs(
|
|
141
|
+
cwd: string,
|
|
142
|
+
): Array<{ root: string; source: "project" | "package" }> {
|
|
107
143
|
return [
|
|
108
|
-
{ root: PACKAGE_SKILLS_DIR, source: "package" },
|
|
109
|
-
{ root: path.resolve(cwd, "skills"), source: "project" },
|
|
144
|
+
{ root: PACKAGE_SKILLS_DIR, source: "package" }, // ✓ Trusted first
|
|
145
|
+
{ root: path.resolve(cwd, "skills"), source: "project" }, // ⚠️ Override second
|
|
110
146
|
];
|
|
111
147
|
}
|
|
112
148
|
|
|
@@ -120,7 +156,10 @@ interface CachedSkillMarkdown {
|
|
|
120
156
|
|
|
121
157
|
const skillReadCache = new Map<string, CachedSkillMarkdown>();
|
|
122
158
|
|
|
123
|
-
function rememberSkill(
|
|
159
|
+
function rememberSkill(
|
|
160
|
+
key: string,
|
|
161
|
+
value: CachedSkillMarkdown,
|
|
162
|
+
): CachedSkillMarkdown {
|
|
124
163
|
if (skillReadCache.has(key)) skillReadCache.delete(key);
|
|
125
164
|
skillReadCache.set(key, value);
|
|
126
165
|
while (skillReadCache.size > SKILL_CACHE_MAX_ENTRIES) {
|
|
@@ -144,7 +183,12 @@ function cachedSkillFresh(value: CachedSkillMarkdown): boolean {
|
|
|
144
183
|
}
|
|
145
184
|
}
|
|
146
185
|
|
|
147
|
-
function readSkillMarkdown(
|
|
186
|
+
function readSkillMarkdown(
|
|
187
|
+
cwd: string,
|
|
188
|
+
name: string,
|
|
189
|
+
):
|
|
190
|
+
| { path: string; source: "project" | "package"; content: string }
|
|
191
|
+
| undefined {
|
|
148
192
|
if (!isValidSkillName(name)) return undefined;
|
|
149
193
|
const cacheKey = `${path.resolve(cwd)}:${name}`;
|
|
150
194
|
const cached = skillReadCache.get(cacheKey);
|
|
@@ -158,9 +202,14 @@ function readSkillMarkdown(cwd: string, name: string): { path: string; source: "
|
|
|
158
202
|
if (fs.lstatSync(contained).isSymbolicLink()) continue;
|
|
159
203
|
const filePath = resolveRealContainedPath(entry.root, relative);
|
|
160
204
|
const stat = fs.statSync(filePath);
|
|
161
|
-
return rememberSkill(cacheKey, {
|
|
162
|
-
|
|
163
|
-
|
|
205
|
+
return rememberSkill(cacheKey, {
|
|
206
|
+
path: filePath,
|
|
207
|
+
source: entry.source,
|
|
208
|
+
content: fs.readFileSync(filePath, "utf-8"),
|
|
209
|
+
mtimeMs: stat.mtimeMs,
|
|
210
|
+
size: stat.size,
|
|
211
|
+
});
|
|
212
|
+
} catch {}
|
|
164
213
|
}
|
|
165
214
|
return undefined;
|
|
166
215
|
}
|
|
@@ -168,7 +217,9 @@ function readSkillMarkdown(cwd: string, name: string): { path: string; source: "
|
|
|
168
217
|
function frontmatterDescription(content: string): string | undefined {
|
|
169
218
|
const match = /^---\r?\n([\s\S]*?)\r?\n---/.exec(content);
|
|
170
219
|
if (!match) return undefined;
|
|
171
|
-
const line = match[1]
|
|
220
|
+
const line = match[1]
|
|
221
|
+
.split(/\r?\n/)
|
|
222
|
+
.find((entry) => entry.startsWith("description:"));
|
|
172
223
|
return line?.slice("description:".length).trim();
|
|
173
224
|
}
|
|
174
225
|
|
|
@@ -179,8 +230,12 @@ function stripFrontmatter(content: string): string {
|
|
|
179
230
|
function compactSkillContent(content: string): string {
|
|
180
231
|
const body = stripFrontmatter(content);
|
|
181
232
|
if (body.length <= MAX_SKILL_CHARS) return body;
|
|
182
|
-
const preferred =
|
|
183
|
-
|
|
233
|
+
const preferred =
|
|
234
|
+
body.split(/\r?\n## Verification\r?\n/)[0]?.trim() ?? body;
|
|
235
|
+
const truncated =
|
|
236
|
+
preferred.length > MAX_SKILL_CHARS
|
|
237
|
+
? preferred.slice(0, MAX_SKILL_CHARS - 40).trimEnd()
|
|
238
|
+
: preferred;
|
|
184
239
|
return `${truncated}\n\n[skill instructions truncated]`;
|
|
185
240
|
}
|
|
186
241
|
|
|
@@ -197,7 +252,11 @@ export interface RenderedSkillInstructions {
|
|
|
197
252
|
}>;
|
|
198
253
|
}
|
|
199
254
|
|
|
200
|
-
export function renderSkillInstructions(
|
|
255
|
+
export function renderSkillInstructions(
|
|
256
|
+
input: RenderSkillInstructionsInput & {
|
|
257
|
+
runId?: string;
|
|
258
|
+
} = {} as RenderSkillInstructionsInput & { runId?: string },
|
|
259
|
+
): RenderedSkillInstructions {
|
|
201
260
|
const allNames = collectTaskSkillNames(input);
|
|
202
261
|
const names = allNames.slice(0, MAX_SELECTED_SKILLS);
|
|
203
262
|
const overflowCount = Math.max(0, allNames.length - names.length);
|
|
@@ -208,12 +267,18 @@ export function renderSkillInstructions(input: RenderSkillInstructionsInput & {
|
|
|
208
267
|
let omittedCount = overflowCount;
|
|
209
268
|
|
|
210
269
|
// ECC INSTINCT: Get confidence-weighted skills if runId is provided
|
|
211
|
-
let weightedSkills: RenderedSkillInstructions["weightedSkills"]
|
|
270
|
+
let weightedSkills: RenderedSkillInstructions["weightedSkills"];
|
|
212
271
|
if (input.runId) {
|
|
213
272
|
// Register effectiveness hooks once per process
|
|
214
273
|
registerSkillEffectivenessHooks();
|
|
215
|
-
const weighted = getWeightedSkillsForRole(
|
|
216
|
-
|
|
274
|
+
const weighted = getWeightedSkillsForRole(
|
|
275
|
+
input.cwd,
|
|
276
|
+
input.role,
|
|
277
|
+
names,
|
|
278
|
+
input.runId,
|
|
279
|
+
CONFIDENCE_THRESHOLDS.TENTATIVE,
|
|
280
|
+
);
|
|
281
|
+
weightedSkills = weighted.map((w) => ({
|
|
217
282
|
skillId: w.skillId,
|
|
218
283
|
confidence: w.confidence,
|
|
219
284
|
behavior: w.behavior,
|
|
@@ -237,14 +302,30 @@ export function renderSkillInstructions(input: RenderSkillInstructionsInput & {
|
|
|
237
302
|
}
|
|
238
303
|
skillPaths.push(path.dirname(loaded.path));
|
|
239
304
|
const description = frontmatterDescription(loaded.content);
|
|
240
|
-
const source =
|
|
305
|
+
const source =
|
|
306
|
+
loaded.source === "project"
|
|
307
|
+
? `project:skills/${safeName}`
|
|
308
|
+
: `package:skills/${safeName}`;
|
|
241
309
|
|
|
242
310
|
// ECC INSTINCT: Add confidence annotation from weighted skills
|
|
243
|
-
const weighted = weightedSkills?.find(w => w.skillId === name);
|
|
244
|
-
const confidenceNote = weighted
|
|
311
|
+
const weighted = weightedSkills?.find((w) => w.skillId === name);
|
|
312
|
+
const confidenceNote = weighted
|
|
313
|
+
? ` [Confidence: ${(weighted.confidence * 100).toFixed(0)}% — ${weighted.threshold}]`
|
|
314
|
+
: "";
|
|
245
315
|
|
|
246
|
-
const header = [
|
|
247
|
-
|
|
316
|
+
const header = [
|
|
317
|
+
`## ${safeName}`,
|
|
318
|
+
description
|
|
319
|
+
? `Description: ${description}${confidenceNote}`
|
|
320
|
+
: undefined,
|
|
321
|
+
`Source: ${source}`,
|
|
322
|
+
]
|
|
323
|
+
.filter(Boolean)
|
|
324
|
+
.join("\n");
|
|
325
|
+
const rawContent = compactSkillContent(loaded.content);
|
|
326
|
+
// Wrap skill content with provenance markers to help LLMs distinguish skill instructions
|
|
327
|
+
const wrappedContent = `<!-- skill: ${safeName} -->\n${rawContent}\n<!-- end-skill: ${safeName} -->`;
|
|
328
|
+
const section = `${header}\n\n${wrappedContent}`;
|
|
248
329
|
if (!pushSection(section)) omittedCount += 1;
|
|
249
330
|
}
|
|
250
331
|
if (omittedCount > 0) {
|