pi-crew 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +160 -0
- package/README.md +70 -33
- package/docs/issue-29-analysis.md +189 -0
- package/docs/superpowers/plans/2026-06-09-fallow-patterns-adoption.md +1268 -0
- package/package.json +2 -2
- package/src/config/config.ts +106 -15
- package/src/errors.ts +107 -0
- package/src/extension/async-notifier.ts +6 -2
- package/src/extension/crew-cleanup.ts +8 -5
- package/src/extension/management.ts +464 -109
- package/src/extension/register.ts +194 -34
- package/src/extension/registration/commands.ts +2 -2
- package/src/extension/registration/subagent-helpers.ts +2 -2
- package/src/extension/registration/subagent-tools.ts +1 -1
- package/src/extension/registration/team-tool.ts +1 -1
- package/src/extension/registration/viewers.ts +2 -2
- package/src/extension/team-tool/api.ts +3 -3
- package/src/extension/team-tool/cancel.ts +3 -3
- package/src/extension/team-tool/explain.ts +1 -1
- package/src/extension/team-tool/handle-schedule.ts +40 -0
- package/src/extension/team-tool/inspect.ts +3 -3
- package/src/extension/team-tool/lifecycle-actions.ts +4 -4
- package/src/extension/team-tool/respond.ts +2 -2
- package/src/extension/team-tool/run.ts +60 -11
- package/src/extension/team-tool/status.ts +1 -1
- package/src/extension/team-tool.ts +173 -46
- package/src/hooks/registry.ts +77 -13
- package/src/hooks/types.ts +9 -0
- package/src/plugins/plugin-define.ts +6 -0
- package/src/plugins/plugin-registry.ts +32 -0
- package/src/plugins/plugins/index.ts +3 -0
- package/src/plugins/plugins/nextjs.ts +19 -0
- package/src/plugins/plugins/vite.ts +14 -0
- package/src/plugins/plugins/vitest.ts +14 -0
- package/src/runtime/async-runner.ts +35 -7
- package/src/runtime/background-runner.ts +529 -144
- package/src/runtime/chain-parser.ts +5 -1
- package/src/runtime/chain-runner.ts +9 -3
- package/src/runtime/checkpoint.ts +262 -180
- package/src/runtime/child-pi.ts +164 -37
- package/src/runtime/crash-recovery.ts +25 -14
- package/src/runtime/crew-agent-records.ts +2 -0
- package/src/runtime/diagnostic-export.ts +1 -1
- package/src/runtime/dynamic-script-runner.ts +10 -7
- package/src/runtime/foreground-watchdog.ts +1 -1
- package/src/runtime/heartbeat-watcher.ts +19 -2
- package/src/runtime/intercom-bridge.ts +7 -0
- package/src/runtime/iteration-hooks.ts +1 -1
- package/src/runtime/manifest-cache.ts +4 -0
- package/src/runtime/orphan-worker-registry.ts +444 -0
- package/src/runtime/parent-guard.ts +70 -16
- package/src/runtime/pi-args.ts +437 -14
- package/src/runtime/pi-spawn.ts +1 -0
- package/src/runtime/post-checks.ts +1 -1
- package/src/runtime/retry-runner.ts +9 -3
- package/src/runtime/run-tracker.ts +38 -10
- package/src/runtime/sandbox.ts +17 -2
- package/src/runtime/scheduler.ts +25 -2
- package/src/runtime/skill-effectiveness.ts +105 -62
- package/src/runtime/skill-instructions.ts +110 -32
- package/src/runtime/stale-reconciler.ts +310 -69
- package/src/runtime/subagent-manager.ts +251 -57
- package/src/runtime/task-health.ts +76 -0
- package/src/runtime/task-id.ts +20 -0
- package/src/runtime/task-runner/live-executor.ts +1 -1
- package/src/runtime/task-runner/progress.ts +1 -1
- package/src/runtime/task-runner/state-helpers.ts +110 -6
- package/src/runtime/task-runner.ts +92 -70
- package/src/runtime/team-runner.ts +186 -20
- package/src/schema/team-tool-schema.ts +27 -9
- package/src/skills/discover-skills.ts +9 -3
- package/src/state/active-run-registry.ts +170 -38
- package/src/state/artifact-store.ts +25 -13
- package/src/state/atomic-write-v2.ts +86 -0
- package/src/state/atomic-write.ts +346 -55
- package/src/state/blob-store.ts +178 -10
- package/src/state/crew-init.ts +161 -28
- package/src/state/decision-ledger.ts +172 -111
- package/src/state/event-log-rotation.ts +82 -52
- package/src/state/event-log.ts +254 -70
- package/src/state/health-store.ts +71 -0
- package/src/state/locks.ts +102 -20
- package/src/state/mailbox.ts +45 -7
- package/src/state/observation-store.ts +4 -1
- package/src/state/run-graph.ts +141 -130
- package/src/state/run-metrics.ts +24 -8
- package/src/state/state-store.ts +330 -43
- package/src/ui/live-run-sidebar.ts +1 -1
- package/src/ui/loaders.ts +4 -0
- package/src/ui/overlays/agent-picker-overlay.ts +1 -1
- package/src/ui/overlays/mailbox-detail-overlay.ts +1 -1
- package/src/ui/run-action-dispatcher.ts +2 -2
- package/src/ui/run-snapshot-cache.ts +1 -1
- package/src/utils/env-filter.ts +66 -0
- package/src/utils/file-coalescer.ts +9 -1
- package/src/utils/paths.ts +80 -5
- package/src/utils/redaction.ts +6 -1
- package/src/utils/safe-paths.ts +281 -10
- package/src/worktree/cleanup.ts +112 -24
- package/src/worktree/worktree-manager.ts +128 -15
- package/test-bugs-all.mjs +10 -6
- package/test-integration-check.ts +4 -0
package/src/runtime/scheduler.ts
CHANGED
|
@@ -15,11 +15,13 @@ export interface ScheduledJob {
|
|
|
15
15
|
lastStatus?: "success" | "error" | "running";
|
|
16
16
|
nextRun?: string;
|
|
17
17
|
runCount: number;
|
|
18
|
+
/** Run IDs spawned by this job. Used to cancel runs when job is removed. */
|
|
19
|
+
spawnedRunIds?: string[];
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
export type ScheduleChangeEvent =
|
|
21
23
|
| { type: "added"; job: ScheduledJob }
|
|
22
|
-
| { type: "removed"; jobId: string }
|
|
24
|
+
| { type: "removed"; jobId: string; spawnedRunIds?: string[] }
|
|
23
25
|
| { type: "updated"; job: ScheduledJob }
|
|
24
26
|
| { type: "fired"; jobId: string; agentId: string; name: string }
|
|
25
27
|
| { type: "error"; jobId: string; error: string };
|
|
@@ -30,17 +32,21 @@ export class CrewScheduler {
|
|
|
30
32
|
private emit?: (event: ScheduleChangeEvent) => void;
|
|
31
33
|
private executor?: (job: ScheduledJob) => string;
|
|
32
34
|
private finalizer?: (jobId: string, agentId: string) => void;
|
|
35
|
+
private runCancelFn?: (runId: string) => void;
|
|
33
36
|
|
|
34
37
|
start(
|
|
35
38
|
options: {
|
|
36
39
|
emit: (event: ScheduleChangeEvent) => void;
|
|
37
40
|
executor: (job: ScheduledJob) => string;
|
|
38
41
|
finalizer: (jobId: string, agentId: string) => void;
|
|
42
|
+
/** Optional callback to cancel a spawned run by runId. */
|
|
43
|
+
runCancelFn?: (runId: string) => void;
|
|
39
44
|
},
|
|
40
45
|
): void {
|
|
41
46
|
this.emit = options.emit;
|
|
42
47
|
this.executor = options.executor;
|
|
43
48
|
this.finalizer = options.finalizer;
|
|
49
|
+
this.runCancelFn = options.runCancelFn;
|
|
44
50
|
}
|
|
45
51
|
|
|
46
52
|
stop(): void {
|
|
@@ -52,6 +58,7 @@ export class CrewScheduler {
|
|
|
52
58
|
this.emit = undefined;
|
|
53
59
|
this.executor = undefined;
|
|
54
60
|
this.finalizer = undefined;
|
|
61
|
+
this.runCancelFn = undefined;
|
|
55
62
|
}
|
|
56
63
|
|
|
57
64
|
add(job: ScheduledJob): void {
|
|
@@ -61,9 +68,17 @@ export class CrewScheduler {
|
|
|
61
68
|
}
|
|
62
69
|
|
|
63
70
|
remove(id: string): boolean {
|
|
71
|
+
const job = this.jobs.get(id);
|
|
72
|
+
const spawnedRunIds = job?.spawnedRunIds;
|
|
64
73
|
this.disarm(id);
|
|
74
|
+
// Cancel all spawned runs that are still active
|
|
75
|
+
if (spawnedRunIds && this.runCancelFn) {
|
|
76
|
+
for (const runId of spawnedRunIds) {
|
|
77
|
+
try { this.runCancelFn(runId); } catch { /* best-effort */ }
|
|
78
|
+
}
|
|
79
|
+
}
|
|
65
80
|
const ok = this.jobs.delete(id);
|
|
66
|
-
if (ok) this.emit?.({ type: "removed", jobId: id });
|
|
81
|
+
if (ok) this.emit?.({ type: "removed", jobId: id, spawnedRunIds });
|
|
67
82
|
return ok;
|
|
68
83
|
}
|
|
69
84
|
|
|
@@ -82,6 +97,14 @@ export class CrewScheduler {
|
|
|
82
97
|
return [...this.jobs.values()];
|
|
83
98
|
}
|
|
84
99
|
|
|
100
|
+
/** Record a runId spawned by a job. Call this after executor fires. */
|
|
101
|
+
recordSpawnedRun(jobId: string, runId: string): void {
|
|
102
|
+
const job = this.jobs.get(jobId);
|
|
103
|
+
if (!job) return;
|
|
104
|
+
const spawnedRunIds = [...(job.spawnedRunIds ?? []), runId];
|
|
105
|
+
this.jobs.set(jobId, { ...job, spawnedRunIds });
|
|
106
|
+
}
|
|
107
|
+
|
|
85
108
|
private arm(job: ScheduledJob): void {
|
|
86
109
|
if (this.timers.has(job.id)) return;
|
|
87
110
|
if (job.scheduleType === "interval" && job.intervalMs) {
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Skill Effectiveness — ECC INSTINCT/CONFIDENCE Pattern Implementation
|
|
3
|
-
*
|
|
3
|
+
*
|
|
4
4
|
* Implements confidence-weighted skill activation based on ECC's instinct system.
|
|
5
5
|
* Tracks skill activation success and adjusts confidence scores.
|
|
6
|
-
*
|
|
6
|
+
*
|
|
7
7
|
* Based on: docs/distillation/ECC-hooks-instincts.md §2-3 (instinct system, confidence thresholds)
|
|
8
8
|
* Based on: docs/distillation/ECC-10-skills.md §8 (continuous-learning-v2)
|
|
9
|
-
*
|
|
9
|
+
*
|
|
10
10
|
* @module skill-effectiveness
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
14
14
|
import { dirname, join } from "path";
|
|
15
|
+
import { projectCrewRoot } from "../utils/paths.ts";
|
|
15
16
|
import { crewHooks } from "./crew-hooks.ts";
|
|
16
17
|
|
|
17
18
|
/**
|
|
@@ -19,10 +20,10 @@ import { crewHooks } from "./crew-hooks.ts";
|
|
|
19
20
|
* Skills below 0.3 threshold are considered tentative and not enforced.
|
|
20
21
|
*/
|
|
21
22
|
export const CONFIDENCE_THRESHOLDS = {
|
|
22
|
-
TENTATIVE: 0.3,
|
|
23
|
-
MODERATE: 0.5,
|
|
24
|
-
STRONG: 0.7,
|
|
25
|
-
NEAR_CERTAIN: 0.9,
|
|
23
|
+
TENTATIVE: 0.3, // Suggested but not enforced
|
|
24
|
+
MODERATE: 0.5, // Applied when relevant
|
|
25
|
+
STRONG: 0.7, // Auto-approved for application
|
|
26
|
+
NEAR_CERTAIN: 0.9, // Core behavior
|
|
26
27
|
} as const;
|
|
27
28
|
|
|
28
29
|
/**
|
|
@@ -30,12 +31,12 @@ export const CONFIDENCE_THRESHOLDS = {
|
|
|
30
31
|
* From ECC instinct system: 1-2 observations → 0.3, 3-5 → 0.5, etc.
|
|
31
32
|
*/
|
|
32
33
|
export const INITIAL_CONFIDENCE_BY_FREQUENCY: Record<string, number> = {
|
|
33
|
-
"1": 0.3,
|
|
34
|
-
"2": 0.3,
|
|
35
|
-
"3": 0.5,
|
|
34
|
+
"1": 0.3, // 1 observation → tentative
|
|
35
|
+
"2": 0.3, // 2 observations → tentative
|
|
36
|
+
"3": 0.5, // 3 observations → moderate
|
|
36
37
|
"4": 0.5,
|
|
37
38
|
"5": 0.5,
|
|
38
|
-
"6": 0.7,
|
|
39
|
+
"6": 0.7, // 6-10 observations → strong
|
|
39
40
|
"7": 0.7,
|
|
40
41
|
"8": 0.7,
|
|
41
42
|
"9": 0.7,
|
|
@@ -47,8 +48,8 @@ export const INITIAL_CONFIDENCE_BY_FREQUENCY: Record<string, number> = {
|
|
|
47
48
|
* Confidence adjustments per ECC instinct system.
|
|
48
49
|
*/
|
|
49
50
|
export const CONFIDENCE_ADJUSTMENTS = {
|
|
50
|
-
CONFIRMING: 0.05,
|
|
51
|
-
CONTRADICTING: -0.1,
|
|
51
|
+
CONFIRMING: 0.05, // Each confirming observation
|
|
52
|
+
CONTRADICTING: -0.1, // Each contradicting observation
|
|
52
53
|
DECAY_PER_WEEK: -0.02, // Per week without observation
|
|
53
54
|
} as const;
|
|
54
55
|
|
|
@@ -57,24 +58,24 @@ export const CONFIDENCE_ADJUSTMENTS = {
|
|
|
57
58
|
* Skill can be promoted to "strong enforcement" when these are met.
|
|
58
59
|
*/
|
|
59
60
|
export const PROMOTION_GATE_CRITERIA = {
|
|
60
|
-
MIN_CORRECTNESS: 0.8,
|
|
61
|
-
MIN_ACTIVATIONS: 5,
|
|
62
|
-
MIN_AVG_CONFIDENCE: 0.7,
|
|
61
|
+
MIN_CORRECTNESS: 0.8, // 80% pass rate
|
|
62
|
+
MIN_ACTIVATIONS: 5, // Minimum observations before filtering
|
|
63
|
+
MIN_AVG_CONFIDENCE: 0.7, // Average confidence threshold
|
|
63
64
|
} as const;
|
|
64
65
|
|
|
65
66
|
/**
|
|
66
67
|
* Skill activation record - captures each time a skill is used.
|
|
67
68
|
*/
|
|
68
69
|
export interface SkillActivation {
|
|
69
|
-
id: string;
|
|
70
|
-
skillId: string;
|
|
71
|
-
role: string;
|
|
72
|
-
runId: string;
|
|
73
|
-
taskId: string;
|
|
74
|
-
timestamp: string;
|
|
75
|
-
passed: boolean;
|
|
76
|
-
outcome?: string;
|
|
77
|
-
confidence: number;
|
|
70
|
+
id: string; // Unique activation ID
|
|
71
|
+
skillId: string; // Skill identifier (e.g., "verification-before-done")
|
|
72
|
+
role: string; // Role that activated the skill
|
|
73
|
+
runId: string; // Run ID
|
|
74
|
+
taskId: string; // Task ID
|
|
75
|
+
timestamp: string; // ISO timestamp
|
|
76
|
+
passed: boolean; // Whether the skill was successfully applied
|
|
77
|
+
outcome?: string; // Optional outcome description
|
|
78
|
+
confidence: number; // Confidence at time of activation
|
|
78
79
|
}
|
|
79
80
|
|
|
80
81
|
/**
|
|
@@ -85,13 +86,13 @@ export interface SkillMetrics {
|
|
|
85
86
|
totalActivations: number;
|
|
86
87
|
passedActivations: number;
|
|
87
88
|
failedActivations: number;
|
|
88
|
-
passRate: number;
|
|
89
|
-
avgConfidence: number;
|
|
90
|
-
currentConfidence: number;
|
|
89
|
+
passRate: number; // passed / total
|
|
90
|
+
avgConfidence: number; // Rolling average confidence
|
|
91
|
+
currentConfidence: number; // Current confidence score
|
|
91
92
|
trend: "improving" | "stable" | "declining";
|
|
92
|
-
lastActivation?: string;
|
|
93
|
-
firstActivation?: string;
|
|
94
|
-
roleBreakdown: Record<string, number>;
|
|
93
|
+
lastActivation?: string; // ISO timestamp
|
|
94
|
+
firstActivation?: string; // ISO timestamp
|
|
95
|
+
roleBreakdown: Record<string, number>; // Activations per role
|
|
95
96
|
}
|
|
96
97
|
|
|
97
98
|
/**
|
|
@@ -102,35 +103,39 @@ export interface WeightedSkill {
|
|
|
102
103
|
confidence: number;
|
|
103
104
|
threshold: keyof typeof CONFIDENCE_THRESHOLDS;
|
|
104
105
|
behavior: "suggest" | "apply_if_asked" | "apply_auto" | "act_autonomous";
|
|
105
|
-
evidence: string;
|
|
106
|
+
evidence: string; // Evidence for confidence score
|
|
106
107
|
metrics: SkillMetrics;
|
|
107
108
|
}
|
|
108
109
|
|
|
109
110
|
/**
|
|
110
111
|
* Get skill effectiveness storage path.
|
|
112
|
+
* Uses projectCrewRoot() so the path honours the `.pi/teams/` fallback
|
|
113
|
+
* for `.pi`-based projects (see issue #29).
|
|
111
114
|
*/
|
|
112
|
-
function getSkillMetricsPath(runId: string): string {
|
|
115
|
+
function getSkillMetricsPath(cwd: string, runId: string): string {
|
|
113
116
|
return join(
|
|
114
|
-
|
|
115
|
-
|
|
117
|
+
projectCrewRoot(cwd),
|
|
118
|
+
`state/runs/${runId}/skill-metrics.jsonl`,
|
|
116
119
|
);
|
|
117
120
|
}
|
|
118
121
|
|
|
119
122
|
/**
|
|
120
123
|
* Get skill activations path.
|
|
124
|
+
* Uses projectCrewRoot() so the path honours the `.pi/teams/` fallback
|
|
125
|
+
* for `.pi`-based projects (see issue #29).
|
|
121
126
|
*/
|
|
122
|
-
function getSkillActivationsPath(runId: string): string {
|
|
127
|
+
function getSkillActivationsPath(cwd: string, runId: string): string {
|
|
123
128
|
return join(
|
|
124
|
-
|
|
125
|
-
|
|
129
|
+
projectCrewRoot(cwd),
|
|
130
|
+
`state/runs/${runId}/skill-activations.jsonl`,
|
|
126
131
|
);
|
|
127
132
|
}
|
|
128
133
|
|
|
129
134
|
/**
|
|
130
135
|
* Ensure directory exists for skill metrics.
|
|
131
136
|
*/
|
|
132
|
-
function ensureSkillMetricsDir(runId: string): void {
|
|
133
|
-
const dir = dirname(getSkillMetricsPath(runId));
|
|
137
|
+
function ensureSkillMetricsDir(cwd: string, runId: string): void {
|
|
138
|
+
const dir = dirname(getSkillMetricsPath(cwd, runId));
|
|
134
139
|
if (!existsSync(dir)) {
|
|
135
140
|
mkdirSync(dir, { recursive: true });
|
|
136
141
|
}
|
|
@@ -163,7 +168,9 @@ export function adjustConfidence(current: number, passed: boolean): number {
|
|
|
163
168
|
export function applyDecay(current: number, lastActivation?: string): number {
|
|
164
169
|
if (!lastActivation) return current;
|
|
165
170
|
|
|
166
|
-
const daysSince =
|
|
171
|
+
const daysSince =
|
|
172
|
+
(Date.now() - new Date(lastActivation).getTime()) /
|
|
173
|
+
(1000 * 60 * 60 * 24);
|
|
167
174
|
const decayWeeks = Math.floor(daysSince / 7);
|
|
168
175
|
const decay = decayWeeks * CONFIDENCE_ADJUSTMENTS.DECAY_PER_WEEK;
|
|
169
176
|
|
|
@@ -173,8 +180,11 @@ export function applyDecay(current: number, lastActivation?: string): number {
|
|
|
173
180
|
/**
|
|
174
181
|
* Determine behavior based on confidence threshold.
|
|
175
182
|
*/
|
|
176
|
-
export function confidenceToBehavior(
|
|
177
|
-
|
|
183
|
+
export function confidenceToBehavior(
|
|
184
|
+
confidence: number,
|
|
185
|
+
): WeightedSkill["behavior"] {
|
|
186
|
+
if (confidence >= CONFIDENCE_THRESHOLDS.NEAR_CERTAIN)
|
|
187
|
+
return "act_autonomous";
|
|
178
188
|
if (confidence >= CONFIDENCE_THRESHOLDS.STRONG) return "apply_auto";
|
|
179
189
|
if (confidence >= CONFIDENCE_THRESHOLDS.MODERATE) return "apply_if_asked";
|
|
180
190
|
return "suggest";
|
|
@@ -183,7 +193,9 @@ export function confidenceToBehavior(confidence: number): WeightedSkill["behavio
|
|
|
183
193
|
/**
|
|
184
194
|
* Determine threshold name from confidence.
|
|
185
195
|
*/
|
|
186
|
-
export function confidenceToThreshold(
|
|
196
|
+
export function confidenceToThreshold(
|
|
197
|
+
confidence: number,
|
|
198
|
+
): keyof typeof CONFIDENCE_THRESHOLDS {
|
|
187
199
|
if (confidence >= CONFIDENCE_THRESHOLDS.NEAR_CERTAIN) return "NEAR_CERTAIN";
|
|
188
200
|
if (confidence >= CONFIDENCE_THRESHOLDS.STRONG) return "STRONG";
|
|
189
201
|
if (confidence >= CONFIDENCE_THRESHOLDS.TENTATIVE) return "MODERATE";
|
|
@@ -195,11 +207,12 @@ export function confidenceToThreshold(confidence: number): keyof typeof CONFIDEN
|
|
|
195
207
|
* Appends to the run's skill-activations.jsonl for learning.
|
|
196
208
|
*/
|
|
197
209
|
export function recordSkillActivation(
|
|
210
|
+
cwd: string,
|
|
198
211
|
activation: SkillActivation,
|
|
199
212
|
): SkillActivation {
|
|
200
|
-
ensureSkillMetricsDir(activation.runId);
|
|
213
|
+
ensureSkillMetricsDir(cwd, activation.runId);
|
|
201
214
|
|
|
202
|
-
const path = getSkillActivationsPath(activation.runId);
|
|
215
|
+
const path = getSkillActivationsPath(cwd, activation.runId);
|
|
203
216
|
const line = JSON.stringify(activation) + "\n";
|
|
204
217
|
writeFileSync(path, line, { flag: "a", encoding: "utf-8" });
|
|
205
218
|
|
|
@@ -209,8 +222,11 @@ export function recordSkillActivation(
|
|
|
209
222
|
/**
|
|
210
223
|
* Get all skill activations for a run.
|
|
211
224
|
*/
|
|
212
|
-
export function getSkillActivations(
|
|
213
|
-
|
|
225
|
+
export function getSkillActivations(
|
|
226
|
+
cwd: string,
|
|
227
|
+
runId: string,
|
|
228
|
+
): SkillActivation[] {
|
|
229
|
+
const path = getSkillActivationsPath(cwd, runId);
|
|
214
230
|
|
|
215
231
|
if (!existsSync(path)) {
|
|
216
232
|
return [];
|
|
@@ -256,11 +272,13 @@ export function computeSkillMetrics(
|
|
|
256
272
|
skillActivations.reduce((sum, a) => sum + a.confidence, 0) /
|
|
257
273
|
skillActivations.length;
|
|
258
274
|
const currentConfidence =
|
|
259
|
-
skillActivations[skillActivations.length - 1]?.confidence ??
|
|
275
|
+
skillActivations[skillActivations.length - 1]?.confidence ??
|
|
276
|
+
avgConfidence;
|
|
260
277
|
|
|
261
278
|
// Compute trend from last 5 activations
|
|
262
279
|
const recent = skillActivations.slice(-5);
|
|
263
|
-
const recentPassRate =
|
|
280
|
+
const recentPassRate =
|
|
281
|
+
recent.filter((a) => a.passed).length / recent.length;
|
|
264
282
|
const earlier = skillActivations.slice(0, -5);
|
|
265
283
|
const earlierPassRate =
|
|
266
284
|
earlier.length > 0
|
|
@@ -282,7 +300,8 @@ export function computeSkillMetrics(
|
|
|
282
300
|
}
|
|
283
301
|
|
|
284
302
|
// Apply decay if not observed recently
|
|
285
|
-
const lastActivation =
|
|
303
|
+
const lastActivation =
|
|
304
|
+
skillActivations[skillActivations.length - 1]?.timestamp;
|
|
286
305
|
const decayedConfidence = applyDecay(currentConfidence, lastActivation);
|
|
287
306
|
|
|
288
307
|
return {
|
|
@@ -315,10 +334,13 @@ export function evaluatePromotionGate(metrics: SkillMetrics): {
|
|
|
315
334
|
reason: string;
|
|
316
335
|
} {
|
|
317
336
|
const criteria = {
|
|
318
|
-
correctness:
|
|
319
|
-
|
|
337
|
+
correctness:
|
|
338
|
+
metrics.passRate >= PROMOTION_GATE_CRITERIA.MIN_CORRECTNESS,
|
|
339
|
+
evidence:
|
|
340
|
+
metrics.totalActivations >= PROMOTION_GATE_CRITERIA.MIN_ACTIVATIONS,
|
|
320
341
|
rollback: metrics.trend !== "declining",
|
|
321
|
-
encoding:
|
|
342
|
+
encoding:
|
|
343
|
+
metrics.avgConfidence >= PROMOTION_GATE_CRITERIA.MIN_AVG_CONFIDENCE,
|
|
322
344
|
};
|
|
323
345
|
|
|
324
346
|
const allPassed = Object.values(criteria).every(Boolean);
|
|
@@ -341,12 +363,13 @@ export function evaluatePromotionGate(metrics: SkillMetrics): {
|
|
|
341
363
|
* Filters by minimum confidence threshold.
|
|
342
364
|
*/
|
|
343
365
|
export function getWeightedSkillsForRole(
|
|
366
|
+
cwd: string,
|
|
344
367
|
role: string,
|
|
345
368
|
skillIds: string[],
|
|
346
369
|
runId: string,
|
|
347
370
|
minConfidence: number = CONFIDENCE_THRESHOLDS.TENTATIVE,
|
|
348
371
|
): WeightedSkill[] {
|
|
349
|
-
const activations = getSkillActivations(runId);
|
|
372
|
+
const activations = getSkillActivations(cwd, runId);
|
|
350
373
|
|
|
351
374
|
return skillIds
|
|
352
375
|
.map((skillId) => {
|
|
@@ -376,12 +399,19 @@ export function getWeightedSkillsForRole(
|
|
|
376
399
|
*/
|
|
377
400
|
/** @internal */
|
|
378
401
|
function filterSkillsByConfidence(
|
|
402
|
+
cwd: string,
|
|
379
403
|
skillIds: string[],
|
|
380
404
|
runId: string,
|
|
381
405
|
threshold: keyof typeof CONFIDENCE_THRESHOLDS = "MODERATE",
|
|
382
406
|
): WeightedSkill[] {
|
|
383
407
|
const minConfidence = CONFIDENCE_THRESHOLDS[threshold];
|
|
384
|
-
return getWeightedSkillsForRole(
|
|
408
|
+
return getWeightedSkillsForRole(
|
|
409
|
+
cwd,
|
|
410
|
+
"global",
|
|
411
|
+
skillIds,
|
|
412
|
+
runId,
|
|
413
|
+
minConfidence,
|
|
414
|
+
);
|
|
385
415
|
}
|
|
386
416
|
|
|
387
417
|
/**
|
|
@@ -415,7 +445,11 @@ export function registerSkillEffectivenessHooks(): void {
|
|
|
415
445
|
passed: success,
|
|
416
446
|
confidence: computeInitialConfidence(1),
|
|
417
447
|
};
|
|
418
|
-
|
|
448
|
+
// cwd comes from the event payload (set by callers) so that the
|
|
449
|
+
// activation lands in the correct .pi/teams/ or .crew/state/runs/
|
|
450
|
+
// (see issue #29).
|
|
451
|
+
const eventCwd = (data?.cwd as string) ?? process.cwd();
|
|
452
|
+
recordSkillActivation(eventCwd, activation);
|
|
419
453
|
}
|
|
420
454
|
});
|
|
421
455
|
|
|
@@ -434,10 +468,11 @@ export function registerSkillEffectivenessHooks(): void {
|
|
|
434
468
|
*/
|
|
435
469
|
/** @internal */
|
|
436
470
|
function generateSkillEffectivenessReport(
|
|
471
|
+
cwd: string,
|
|
437
472
|
runId: string,
|
|
438
473
|
skillIds: string[],
|
|
439
474
|
): string {
|
|
440
|
-
const activations = getSkillActivations(runId);
|
|
475
|
+
const activations = getSkillActivations(cwd, runId);
|
|
441
476
|
const lines: string[] = [
|
|
442
477
|
`# Skill Effectiveness Report: ${runId}`,
|
|
443
478
|
"",
|
|
@@ -459,13 +494,21 @@ function generateSkillEffectivenessReport(
|
|
|
459
494
|
const gate = evaluatePromotionGate(metrics);
|
|
460
495
|
|
|
461
496
|
lines.push(`### ${skillId}`);
|
|
462
|
-
lines.push(
|
|
463
|
-
|
|
497
|
+
lines.push(
|
|
498
|
+
`- **Confidence**: ${metrics.currentConfidence.toFixed(2)} (${metrics.trend})`,
|
|
499
|
+
);
|
|
500
|
+
lines.push(
|
|
501
|
+
`- **Pass Rate**: ${(metrics.passRate * 100).toFixed(1)}% (${metrics.passedActivations}/${metrics.totalActivations})`,
|
|
502
|
+
);
|
|
464
503
|
lines.push(`- **Avg Confidence**: ${metrics.avgConfidence.toFixed(2)}`);
|
|
465
|
-
lines.push(
|
|
504
|
+
lines.push(
|
|
505
|
+
`- **Promotion Gate**: ${gate.passed ? "PASSED ✅" : "NOT MET"}`,
|
|
506
|
+
);
|
|
466
507
|
|
|
467
508
|
if (Object.keys(metrics.roleBreakdown).length > 0) {
|
|
468
|
-
lines.push(
|
|
509
|
+
lines.push(
|
|
510
|
+
`- **By Role**: ${JSON.stringify(metrics.roleBreakdown)}`,
|
|
511
|
+
);
|
|
469
512
|
}
|
|
470
513
|
|
|
471
514
|
lines.push("");
|