pi-crew 0.9.41 → 0.9.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +106 -0
- package/dist/build-meta.json +7087 -6726
- package/dist/index.mjs +43043 -42426
- package/dist/index.mjs.map +4 -4
- package/docs/migration/atomic-write-v2-migration.md +2 -0
- package/docs/superpowers/plans/2026-06-09-fallow-patterns-adoption.md +2 -0
- package/package.json +4 -2
- package/scripts/README.md +6 -0
- package/scripts/check-all-skills.ts +3 -3
- package/scripts/dev-runner.mjs +106 -0
- package/scripts/postinstall.mjs +6 -2
- package/scripts/verify-skill.ts +2 -2
- package/src/config/config.ts +19 -0
- package/src/config/drift-detector.ts +97 -15
- package/src/config/resilient-parser.ts +4 -4
- package/src/extension/crew-cleanup.ts +17 -3
- package/src/extension/crew-vibes/index.ts +1 -1
- package/src/extension/cross-extension-rpc.ts +18 -5
- package/src/extension/management.ts +6 -5
- package/src/extension/register.ts +58 -1573
- package/src/extension/registration/command-registration.ts +57 -0
- package/src/extension/registration/commands.ts +2 -1
- package/src/extension/registration/context-builder.ts +131 -0
- package/src/extension/registration/crash-recovery-cache.ts +54 -0
- package/src/extension/registration/foreground-run-controller.ts +210 -0
- package/src/extension/registration/hook-registration.ts +119 -0
- package/src/extension/registration/lazy-configurers.ts +124 -0
- package/src/extension/registration/lifecycle-handlers.ts +731 -0
- package/src/extension/registration/observability.ts +33 -4
- package/src/extension/registration/registration-types.ts +149 -0
- package/src/extension/registration/runtime-cleanup.ts +230 -0
- package/src/extension/registration/subagent-manager-setup.ts +211 -0
- package/src/extension/registration/tool-registration.ts +50 -0
- package/src/extension/registration/wire-cross-extension.ts +37 -0
- package/src/extension/rpc-hmac.ts +21 -1
- package/src/extension/run-import.ts +3 -3
- package/src/extension/team-tool/auto-summarize.ts +4 -4
- package/src/extension/team-tool/cancel.ts +5 -5
- package/src/extension/team-tool/chain-executor.ts +5 -4
- package/src/extension/team-tool/context.ts +11 -1
- package/src/extension/team-tool/doctor.ts +2 -1
- package/src/extension/team-tool/goal-wrap.ts +2 -2
- package/src/extension/team-tool/goal.ts +1 -1
- package/src/extension/team-tool/handle-schedule.ts +25 -36
- package/src/extension/team-tool/orchestrate.ts +0 -1
- package/src/extension/team-tool/parallel-dispatch.ts +5 -0
- package/src/extension/team-tool/run.ts +220 -214
- package/src/extension/team-tool/workflow-manage.ts +1 -1
- package/src/extension/team-tool.ts +64 -16
- package/src/hooks/registry.ts +7 -2
- package/src/observability/correlation.ts +4 -3
- package/src/observability/event-bus.ts +10 -1
- package/src/observability/event-to-metric.ts +8 -7
- package/src/observability/exporters/adapter.ts +6 -9
- package/src/observability/exporters/otlp-exporter.ts +81 -0
- package/src/observability/metric-retention.ts +4 -0
- package/src/observability/metric-sink.ts +14 -4
- package/src/runtime/adaptive-plan.ts +1 -1
- package/src/runtime/async-runner.ts +13 -4
- package/src/runtime/background-runner.ts +32 -23
- package/src/runtime/capability-inventory.ts +1 -1
- package/src/runtime/chain-runner.ts +5 -5
- package/src/runtime/checkpoint.ts +6 -16
- package/src/runtime/child-pi.ts +22 -2
- package/src/runtime/coalesce-tasks.ts +1 -1
- package/src/runtime/crash-recovery.ts +63 -47
- package/src/runtime/delivery-coordinator.ts +1 -1
- package/src/runtime/deterministic-ast.ts +2 -2
- package/src/runtime/diagnostic-export.ts +2 -1
- package/src/runtime/foreground-control.ts +3 -2
- package/src/runtime/goal-loop-runner.ts +6 -2
- package/src/runtime/goal-state-store.ts +1 -1
- package/src/runtime/handoff-manager.ts +0 -2
- package/src/runtime/intercom-bridge.ts +0 -2
- package/src/runtime/live-agent-manager.ts +12 -5
- package/src/runtime/live-extension-bridge.ts +0 -2
- package/src/runtime/live-session-runtime.ts +65 -57
- package/src/runtime/manifest-cache.ts +59 -0
- package/src/runtime/mcp-proxy.ts +1 -1
- package/src/runtime/orphan-worker-registry.ts +3 -3
- package/src/runtime/output-validator.ts +1 -1
- package/src/runtime/path-overlap.ts +1 -1
- package/src/runtime/pi-args.ts +3 -3
- package/src/runtime/pi-json-output.ts +2 -2
- package/src/runtime/pipeline-runner.ts +1 -4
- package/src/runtime/post-checks.ts +0 -1
- package/src/runtime/retry-runner.ts +1 -1
- package/src/runtime/run-coalesced-task-group.ts +0 -13
- package/src/runtime/scheduler.ts +2 -0
- package/src/runtime/session-resources.ts +1 -1
- package/src/runtime/settings-store.ts +31 -1
- package/src/runtime/skill-effectiveness.ts +2 -2
- package/src/runtime/stale-reconciler.ts +23 -53
- package/src/runtime/subagent-manager.ts +19 -3
- package/src/runtime/task-output-context.ts +2 -1
- package/src/runtime/task-runner/prompt-builder.ts +1 -1
- package/src/runtime/task-runner/run-projection.ts +3 -1
- package/src/runtime/task-runner/state-helpers.ts +13 -1
- package/src/runtime/task-runner.ts +31 -6
- package/src/runtime/team-runner.ts +3 -2
- package/src/runtime/verification-gates.ts +54 -5
- package/src/schema/config-schema.ts +19 -1
- package/src/schema/team-tool-schema.ts +2 -0
- package/src/skills/discover-skills.ts +25 -15
- package/src/state/active-run-registry.ts +0 -1
- package/src/state/atomic-write.ts +37 -11
- package/src/state/blob-store.ts +3 -9
- package/src/state/contracts.ts +5 -1
- package/src/state/crew-init.ts +3 -2
- package/src/state/decision-ledger.ts +2 -2
- package/src/state/event-log-rotation.ts +1 -1
- package/src/state/event-log.ts +3 -3
- package/src/state/gitignore-manager.ts +2 -1
- package/src/state/health-store.ts +5 -2
- package/src/state/instinct-store.ts +11 -2
- package/src/state/mailbox.ts +3 -4
- package/src/state/observation-store.ts +1 -13
- package/src/state/run-cache.ts +12 -12
- package/src/state/run-graph.ts +2 -1
- package/src/state/run-metrics.ts +1 -1
- package/src/state/schedule.ts +7 -1
- package/src/state/state-store.ts +9 -3
- package/src/state/tiered-eval.ts +1 -1
- package/src/ui/dynamic-border.ts +13 -3
- package/src/ui/live-conversation-overlay.ts +7 -27
- package/src/ui/live-duration.ts +2 -2
- package/src/ui/live-run-sidebar.ts +56 -32
- package/src/ui/mascot.ts +19 -13
- package/src/ui/powerbar-publisher.ts +244 -178
- package/src/ui/render-coalescer.ts +44 -3
- package/src/ui/render-diff.ts +2 -2
- package/src/ui/run-dashboard.ts +122 -21
- package/src/ui/run-event-bus.ts +66 -0
- package/src/ui/run-snapshot-cache.ts +84 -81
- package/src/ui/settings-overlay.ts +3 -3
- package/src/ui/status-colors.ts +24 -0
- package/src/ui/tool-progress-formatter.ts +1 -1
- package/src/ui/tool-render.ts +3 -3
- package/src/ui/transcript-cache.ts +5 -10
- package/src/ui/transcript-entries.ts +68 -72
- package/src/ui/transcript-viewer.ts +54 -14
- package/src/ui/widget/index.ts +26 -15
- package/src/ui/widget/widget-model.ts +18 -9
- package/src/ui/widget/widget-renderer.ts +2 -5
- package/src/utils/fingerprint.ts +1 -1
- package/src/utils/project-detector.ts +1 -1
- package/src/state/atomic-write-v2.ts +0 -108
|
@@ -14,6 +14,7 @@ export interface CrewEvent {
|
|
|
14
14
|
type CrewEventListener = (event: CrewEvent) => void;
|
|
15
15
|
|
|
16
16
|
class EventBus {
|
|
17
|
+
private static readonly MAX_LISTENERS_PER_EVENT = 100;
|
|
17
18
|
private listeners = new Map<CrewEventType, Set<CrewEventListener>>();
|
|
18
19
|
private static _instance?: EventBus;
|
|
19
20
|
|
|
@@ -54,7 +55,15 @@ class EventBus {
|
|
|
54
55
|
if (!this.listeners.has(type)) {
|
|
55
56
|
this.listeners.set(type, new Set());
|
|
56
57
|
}
|
|
57
|
-
this.listeners.get(type)
|
|
58
|
+
const set = this.listeners.get(type)!;
|
|
59
|
+
if (set.size >= EventBus.MAX_LISTENERS_PER_EVENT) {
|
|
60
|
+
logInternalError(
|
|
61
|
+
"event-bus.listener-cap",
|
|
62
|
+
new Error(`Max ${EventBus.MAX_LISTENERS_PER_EVENT} listeners reached for event type '${type}'`),
|
|
63
|
+
`type=${type} currentCount=${set.size}`,
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
set.add(listener);
|
|
58
67
|
|
|
59
68
|
return () => {
|
|
60
69
|
this.listeners.get(type)?.delete(listener);
|
|
@@ -37,7 +37,7 @@ export function wireEventToMetrics(events: ExtensionAPI["events"] | undefined, r
|
|
|
37
37
|
const taskCount = registry.counter("crew.task.count", "Total tasks by status");
|
|
38
38
|
const subagentCount = registry.counter("crew.subagent.count", "Total subagent records by status");
|
|
39
39
|
const mailboxCount = registry.counter("crew.mailbox.count", "Total mailbox messages by direction");
|
|
40
|
-
const retryAttemptCount = registry.counter("crew.task.retry_attempt_total", "
|
|
40
|
+
const retryAttemptCount = registry.counter("crew.task.retry_attempt_total", "Total retry attempts");
|
|
41
41
|
const deadletterCount = registry.counter("crew.task.deadletter_total", "Deadletter triggers by reason");
|
|
42
42
|
const overflowCount = registry.counter("crew.task.overflow_phase_total", "Overflow recovery phase transitions");
|
|
43
43
|
const supervisorContactCount = registry.counter("crew.task.supervisor_contact_total", "Supervisor contact requests by reason");
|
|
@@ -89,12 +89,12 @@ export function wireEventToMetrics(events: ExtensionAPI["events"] | undefined, r
|
|
|
89
89
|
[
|
|
90
90
|
"crew.task.retry_attempt",
|
|
91
91
|
(data) => {
|
|
92
|
-
const item = recordValue(data);
|
|
93
92
|
taskCount.inc({ status: "retry" });
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
93
|
+
// Per-run/task detail lives in the event log (crew.task.retry_attempt).
|
|
94
|
+
// Using runId/taskId as METRIC labels would blow up cardinality (each unique
|
|
95
|
+
// run+task pair = a new series, silently LRU-evicted at MAX_LABEL_COMBINATIONS
|
|
96
|
+
// = 10_000), making the counter unreliable for aggregation.
|
|
97
|
+
retryAttemptCount.inc({});
|
|
98
98
|
},
|
|
99
99
|
],
|
|
100
100
|
[
|
|
@@ -121,8 +121,9 @@ export function wireEventToMetrics(events: ExtensionAPI["events"] | undefined, r
|
|
|
121
121
|
(data) => {
|
|
122
122
|
const item = recordValue(data);
|
|
123
123
|
supervisorContactCount.inc({
|
|
124
|
+
// reason is low-cardinality; taskId is NOT (unique per task) — dropped to
|
|
125
|
+
// avoid cardinality blowup (OBS-1). Per-task detail is in the event log.
|
|
124
126
|
reason: stringValue(item.reason, "unknown"),
|
|
125
|
-
taskId: stringValue(item.taskId, "unknown"),
|
|
126
127
|
});
|
|
127
128
|
},
|
|
128
129
|
],
|
|
@@ -3,7 +3,7 @@ import type { MetricSnapshot } from "../metrics-primitives.ts";
|
|
|
3
3
|
export interface MetricExporter {
|
|
4
4
|
name: string;
|
|
5
5
|
push(snapshots: MetricSnapshot[]): Promise<void>;
|
|
6
|
-
dispose(): void
|
|
6
|
+
dispose(): Promise<void>;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
export class CompositeExporter implements MetricExporter {
|
|
@@ -18,13 +18,10 @@ export class CompositeExporter implements MetricExporter {
|
|
|
18
18
|
await Promise.allSettled(this.exporters.map((exporter) => exporter.push(snapshots)));
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
dispose(): void {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
// Best-effort cleanup; one exporter failing shouldn't prevent others.
|
|
27
|
-
}
|
|
28
|
-
}
|
|
21
|
+
async dispose(): Promise<void> {
|
|
22
|
+
// OBS-3: await allSettled so an async exporter cleanup (e.g. OTLPExporter awaiting
|
|
23
|
+
// an in-flight HTTP push) isn't cut short. Interface contract changed from `void`
|
|
24
|
+
// to `Promise<void>` because exporters may have async teardown.
|
|
25
|
+
await Promise.allSettled(this.exporters.map((exporter) => exporter.dispose()));
|
|
29
26
|
}
|
|
30
27
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { lookup } from "node:dns/promises";
|
|
1
2
|
import { promisify } from "node:util";
|
|
2
3
|
import { gzip } from "node:zlib";
|
|
3
4
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
@@ -100,6 +101,76 @@ export function validateEndpoint(endpoint: string): void {
|
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
103
|
|
|
104
|
+
/**
|
|
105
|
+
* CFG-6: classify a single IP address (IPv4 or IPv6) as private / reserved /
|
|
106
|
+
* loopback / link-local. Used by the runtime DNS-rebinding guard below to
|
|
107
|
+
* verify the addresses the endpoint hostname resolves to. Returns true for
|
|
108
|
+
* addresses that must never receive a metrics push. The full literal checks
|
|
109
|
+
* in `validateEndpoint` use the same predicates for direct IP targets.
|
|
110
|
+
*/
|
|
111
|
+
export function isPrivateIpAddress(address: string): boolean {
|
|
112
|
+
const lower = address.toLowerCase();
|
|
113
|
+
// IPv4 literal
|
|
114
|
+
const ipv4 = lower.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
115
|
+
if (ipv4) {
|
|
116
|
+
const o0 = Number(ipv4[1]);
|
|
117
|
+
const o1 = Number(ipv4[2]);
|
|
118
|
+
if (o0 === 127 || o0 === 0) return true;
|
|
119
|
+
if (o0 === 10) return true;
|
|
120
|
+
if (o0 === 172 && o1 >= 16 && o1 <= 31) return true;
|
|
121
|
+
if (o0 === 192 && o1 === 168) return true;
|
|
122
|
+
if (o0 === 169 && o1 === 254) return true;
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
// IPv6 (without surrounding brackets, as `dns.lookup` returns them)
|
|
126
|
+
if (lower === "::1" || lower === "::") return true;
|
|
127
|
+
if (lower.startsWith("fd") || lower.startsWith("fc")) return true;
|
|
128
|
+
if (lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb")) return true;
|
|
129
|
+
if (lower.startsWith("fec") || lower.startsWith("fed") || lower.startsWith("fee") || lower.startsWith("fef")) return true;
|
|
130
|
+
if (lower.startsWith("ff")) return true;
|
|
131
|
+
if (lower.startsWith("::ffff:")) return true;
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* CFG-6: DNS-rebinding guard. `validateEndpoint` runs once at exporter
|
|
137
|
+
* construction and only inspects the literal hostname; an attacker who
|
|
138
|
+
* controls the DNS for a public hostname can rebind it to a private IP
|
|
139
|
+
* (e.g. 169.254.169.254 cloud metadata) between construction and the actual
|
|
140
|
+
* fetch. Re-resolve the hostname at push time and reject if any returned
|
|
141
|
+
* address falls into a private/reserved range. Resolution failures are
|
|
142
|
+
* allowed to fall through so the underlying `fetch` surfaces the error.
|
|
143
|
+
*/
|
|
144
|
+
export async function assertResolvedAddressSafe(endpoint: string): Promise<void> {
|
|
145
|
+
let url: URL;
|
|
146
|
+
try {
|
|
147
|
+
url = new URL(endpoint);
|
|
148
|
+
} catch {
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return;
|
|
152
|
+
const hostname = url.hostname.toLowerCase();
|
|
153
|
+
if (!hostname) return;
|
|
154
|
+
// Locals and literal IPs are covered by `validateEndpoint`; skip the DNS
|
|
155
|
+
// round-trip so we don't double-report errors the sync check already
|
|
156
|
+
// produces.
|
|
157
|
+
if (hostname === "localhost" || hostname.endsWith(".localhost")) return;
|
|
158
|
+
if (hostname.startsWith("[")) return;
|
|
159
|
+
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(hostname)) return;
|
|
160
|
+
let addresses: { address: string }[];
|
|
161
|
+
try {
|
|
162
|
+
addresses = await lookup(hostname, { all: true, verbatim: true });
|
|
163
|
+
} catch {
|
|
164
|
+
// Let the fetch fail with a clearer error from the network layer.
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
for (const { address } of addresses) {
|
|
168
|
+
if (isPrivateIpAddress(address)) {
|
|
169
|
+
throw new Error(`OTLP endpoint resolves to private/reserved address ${address} (DNS rebinding blocked): ${endpoint}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
103
174
|
// FIX (Round 15): Cap the number of snapshots per push to prevent OOM when
|
|
104
175
|
// the metric registry has grown large. The OTLP HTTP spec allows many metrics
|
|
105
176
|
// in one payload, but a single push > 10_000 metrics would balloon the
|
|
@@ -210,6 +281,16 @@ export class OTLPExporter implements MetricExporter {
|
|
|
210
281
|
|
|
211
282
|
async push(snapshots: MetricSnapshot[]): Promise<void> {
|
|
212
283
|
try {
|
|
284
|
+
// CFG-6: re-validate the endpoint after DNS resolution to block
|
|
285
|
+
// rebinding from a public hostname to a private/metadata IP
|
|
286
|
+
// between construction time and the actual fetch. Failures are
|
|
287
|
+
// logged and the push is skipped — no metrics reach the bad host.
|
|
288
|
+
try {
|
|
289
|
+
await assertResolvedAddressSafe(this.opts.endpoint);
|
|
290
|
+
} catch (error) {
|
|
291
|
+
logInternalError("otlp-export-ssrf", error);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
213
294
|
// FIX (Round 15): Cap snapshots to a safe size to avoid OOM and
|
|
214
295
|
// oversized HTTP payloads. Log a warning if we are truncating.
|
|
215
296
|
let toSend = snapshots;
|
|
@@ -29,6 +29,10 @@ export class TimeWindowedCounter {
|
|
|
29
29
|
this.prune();
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
// PERF (OBS-5): This method is O(N) per call because it linearly scans
|
|
33
|
+
// all events and calls labelKey() (which uses JSON.stringify) on each.
|
|
34
|
+
// A more efficient aggregation (e.g., pre-aggregated label buckets or a
|
|
35
|
+
// sliding-window histogram) is planned to replace this hot path.
|
|
32
36
|
count(labels: MetricLabels = {}, durationMs = this.windowMs): number {
|
|
33
37
|
const now = this.now();
|
|
34
38
|
this.pruneAt(now);
|
|
@@ -6,7 +6,7 @@ import type { MetricRegistry } from "./metric-registry.ts";
|
|
|
6
6
|
import type { MetricSnapshot } from "./metrics-primitives.ts";
|
|
7
7
|
|
|
8
8
|
export interface MetricSink {
|
|
9
|
-
writeSnapshot(snapshots: MetricSnapshot[]): void
|
|
9
|
+
writeSnapshot(snapshots: MetricSnapshot[]): void | Promise<void>;
|
|
10
10
|
dispose(): void;
|
|
11
11
|
}
|
|
12
12
|
|
|
@@ -54,19 +54,29 @@ export function createMetricFileSink(opts: MetricFileSinkOptions): MetricSink {
|
|
|
54
54
|
fdDate = date;
|
|
55
55
|
return fd;
|
|
56
56
|
};
|
|
57
|
-
const writeSnapshot = (snapshots: MetricSnapshot[]): void => {
|
|
57
|
+
const writeSnapshot = (snapshots: MetricSnapshot[]): Promise<void> => {
|
|
58
58
|
try {
|
|
59
59
|
const now = new Date();
|
|
60
60
|
const date = now.toISOString().slice(0, 10);
|
|
61
61
|
const redacted = redactSecrets(snapshots);
|
|
62
62
|
if (!Array.isArray(redacted)) {
|
|
63
63
|
logInternalError("metric-sink.type", new Error("redactSecrets did not return an array"), `got=${typeof redacted}`);
|
|
64
|
-
return;
|
|
64
|
+
return Promise.resolve();
|
|
65
65
|
}
|
|
66
66
|
const target = ensureFd(date);
|
|
67
|
-
|
|
67
|
+
const line = `${JSON.stringify({ exportedAt: now.toISOString(), snapshots: redacted as MetricSnapshot[] })}\n`;
|
|
68
|
+
// Async write to avoid blocking the main thread on the 60s tick.
|
|
69
|
+
// Return a promise that resolves once the write callback fires so
|
|
70
|
+
// tests (and any callers awaiting) can synchronize on completion.
|
|
71
|
+
return new Promise<void>((resolve) => {
|
|
72
|
+
fs.write(target, line, (err) => {
|
|
73
|
+
if (err) logInternalError("metric-sink.asyncWrite", err);
|
|
74
|
+
resolve();
|
|
75
|
+
});
|
|
76
|
+
});
|
|
68
77
|
} catch (error) {
|
|
69
78
|
logInternalError("metric-sink.write", error);
|
|
79
|
+
return Promise.resolve();
|
|
70
80
|
}
|
|
71
81
|
};
|
|
72
82
|
const timer = setInterval(() => writeSnapshot(opts.registry.snapshot()), opts.intervalMs ?? 60_000);
|
|
@@ -21,7 +21,7 @@ import { saveRunManifestAsync } from "../state/state-store.ts";
|
|
|
21
21
|
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
22
22
|
import type { TeamConfig } from "../teams/team-config.ts";
|
|
23
23
|
import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
|
|
24
|
-
import { getPlanTemplate,
|
|
24
|
+
import { getPlanTemplate, renderPlanTemplate } from "./plan-templates.ts";
|
|
25
25
|
import { refreshTaskGraphQueues } from "./task-graph-scheduler.ts";
|
|
26
26
|
|
|
27
27
|
export interface AdaptivePlanTask {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import { createRequire } from "node:module";
|
|
4
4
|
import * as path from "node:path";
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
6
|
-
import {
|
|
6
|
+
import { appendEventAsync } from "../state/event-log.ts";
|
|
7
7
|
import type { TeamRunManifest } from "../state/types.ts";
|
|
8
8
|
import { WINDOWS_ESSENTIAL_ENV_VARS } from "../utils/env-allowlist.ts";
|
|
9
9
|
import { sanitizeEnvSecrets } from "../utils/env-filter.ts";
|
|
@@ -356,8 +356,17 @@ export async function spawnBackgroundTeamRun(manifest: TeamRunManifest): Promise
|
|
|
356
356
|
);
|
|
357
357
|
// Best-effort: unregister when child exits. Background-runner writes
|
|
358
358
|
// the marker file before it dies, so we unregister on the next
|
|
359
|
-
// cleanup tick.
|
|
360
|
-
//
|
|
359
|
+
// cleanup tick. The child "exit" event may fire after unref() when
|
|
360
|
+
// stdio is piped; register handler defensively.
|
|
361
|
+
if (child.pid) {
|
|
362
|
+
child.on("exit", () => {
|
|
363
|
+
try {
|
|
364
|
+
unregisterWorker(child.pid!);
|
|
365
|
+
} catch {
|
|
366
|
+
/* best-effort */
|
|
367
|
+
}
|
|
368
|
+
});
|
|
369
|
+
}
|
|
361
370
|
}
|
|
362
371
|
|
|
363
372
|
return { pid: child.pid, logPath };
|
|
@@ -2,6 +2,7 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { allAgents, discoverAgents } from "../agents/discover-agents.ts";
|
|
4
4
|
import { loadConfig } from "../config/config.ts";
|
|
5
|
+
import { atomicWriteFile } from "../state/atomic-write.ts";
|
|
5
6
|
import { appendEvent } from "../state/event-log.ts";
|
|
6
7
|
import { withRunLockSync } from "../state/locks.ts";
|
|
7
8
|
import { createRunPaths, loadRunManifestById, saveRunManifestAsync, updateRunStatus } from "../state/state-store.ts";
|
|
@@ -70,7 +71,7 @@ function startHeartbeat(stateRoot: string, eventsPath: string, runId: string): (
|
|
|
70
71
|
const writeHeartbeat = (): void => {
|
|
71
72
|
try {
|
|
72
73
|
const mem = process.memoryUsage();
|
|
73
|
-
|
|
74
|
+
atomicWriteFile(
|
|
74
75
|
heartbeatPath,
|
|
75
76
|
JSON.stringify({
|
|
76
77
|
pid: process.pid,
|
|
@@ -81,7 +82,6 @@ function startHeartbeat(stateRoot: string, eventsPath: string, runId: string): (
|
|
|
81
82
|
rssMb: Math.round(mem.rss / 1024 / 1024),
|
|
82
83
|
},
|
|
83
84
|
}),
|
|
84
|
-
"utf-8",
|
|
85
85
|
);
|
|
86
86
|
} catch {
|
|
87
87
|
/* ignore — best-effort */
|
|
@@ -454,27 +454,36 @@ async function main(): Promise<void> {
|
|
|
454
454
|
}
|
|
455
455
|
}
|
|
456
456
|
// Hook Node.js abort — if process.exit is called with code 1 (uncaught exception, assert failure)
|
|
457
|
-
// we log it before exiting so it appears in background.log
|
|
458
|
-
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
const
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
457
|
+
// we log it before exiting so it appears in background.log.
|
|
458
|
+
// NOTE: process.exit is monkey-patched here (not using process.on('exit')) because
|
|
459
|
+
// we need the log write to happen synchronously before the process actually exits.
|
|
460
|
+
// process.on('exit') handlers run too late for I/O to complete reliably.
|
|
461
|
+
// Wrapped in try/catch to guard against failures in the patch itself.
|
|
462
|
+
try {
|
|
463
|
+
const origExit = process.exit.bind(process);
|
|
464
|
+
// Intercept all exit(code) calls to log them as async.exit events before exiting.
|
|
465
|
+
// This surfaces uncaught exceptions / early exits that would otherwise vanish silently.
|
|
466
|
+
process.exit = ((code?: number | string): never => {
|
|
467
|
+
const runId2 = argValue("--run-id");
|
|
468
|
+
const codeStr = code === undefined ? "<none>" : String(code);
|
|
469
|
+
if (runId2 && manifest.eventsPath) {
|
|
470
|
+
try {
|
|
471
|
+
appendEvent(manifest.eventsPath, {
|
|
472
|
+
type: "async.exit",
|
|
473
|
+
runId: runId2,
|
|
474
|
+
message: `Background runner exit(${codeStr}) pid=${process.pid}`,
|
|
475
|
+
data: { code, pid: process.pid },
|
|
476
|
+
});
|
|
477
|
+
} catch {
|
|
478
|
+
/* best-effort */
|
|
479
|
+
}
|
|
474
480
|
}
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
}
|
|
481
|
+
return origExit(code);
|
|
482
|
+
}) as typeof process.exit;
|
|
483
|
+
} catch {
|
|
484
|
+
// If patching process.exit fails (e.g. frozen process object), continue
|
|
485
|
+
// without the exit log — the run will still proceed normally.
|
|
486
|
+
}
|
|
478
487
|
|
|
479
488
|
// Setup unhandled rejection guard FIRST — must be before any async operations
|
|
480
489
|
// that might produce unhandled rejections during cleanup. Without this, any unhandled
|
|
@@ -632,7 +641,7 @@ async function main(): Promise<void> {
|
|
|
632
641
|
// LAZY: defer dynamic import of ../workflows/discover-workflows.ts to its call site.
|
|
633
642
|
const { allWorkflows, discoverWorkflows } = await import("../workflows/discover-workflows.ts");
|
|
634
643
|
const wf = allWorkflows(discoverWorkflows(manifest.cwd)).find((w) => w.name === manifest.workflow);
|
|
635
|
-
if (
|
|
644
|
+
if (wf?.runtime !== "dynamic" || !wf.dynamicScript)
|
|
636
645
|
throw new Error(
|
|
637
646
|
`runKind="dynamic-workflow" but workflow '${manifest.workflow}' is not dynamic (runId=${manifest.runId})`,
|
|
638
647
|
);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AgentConfig
|
|
1
|
+
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
2
2
|
import { discoverAgents } from "../agents/discover-agents.ts";
|
|
3
3
|
import type { PiTeamsConfig } from "../config/config.ts";
|
|
4
4
|
import { discoverSkills, getLastDiscoveryDiagnostics } from "../skills/discover-skills.ts";
|
|
@@ -301,16 +301,16 @@ export class ChainRunner {
|
|
|
301
301
|
};
|
|
302
302
|
|
|
303
303
|
// Set step type based on matching pattern with type safety
|
|
304
|
-
if (teamMatch
|
|
304
|
+
if (teamMatch?.[1]) {
|
|
305
305
|
parsed.team = this.sanitizeIdentifier(teamMatch[1]);
|
|
306
306
|
}
|
|
307
|
-
if (workflowMatch
|
|
307
|
+
if (workflowMatch?.[1]) {
|
|
308
308
|
parsed.workflow = this.sanitizeIdentifier(workflowMatch[1]);
|
|
309
309
|
}
|
|
310
|
-
if (templateMatch
|
|
310
|
+
if (templateMatch?.[1]) {
|
|
311
311
|
parsed.template = this.sanitizeIdentifier(templateMatch[1]);
|
|
312
312
|
}
|
|
313
|
-
if (inlineMatch
|
|
313
|
+
if (inlineMatch?.[1]) {
|
|
314
314
|
parsed.inlineGoal = this.sanitizeInlineGoal(inlineMatch[1]);
|
|
315
315
|
}
|
|
316
316
|
|
|
@@ -334,7 +334,7 @@ export class ChainRunner {
|
|
|
334
334
|
const timeoutStr = this.extractFlag(step, "timeout");
|
|
335
335
|
if (timeoutStr) {
|
|
336
336
|
const timeoutMs = parseInt(timeoutStr, 10);
|
|
337
|
-
if (!isNaN(timeoutMs) && timeoutMs > 0 && timeoutMs <= 86400000) {
|
|
337
|
+
if (!Number.isNaN(timeoutMs) && timeoutMs > 0 && timeoutMs <= 86400000) {
|
|
338
338
|
parsed.timeout = timeoutMs * 1000; // Convert seconds to ms
|
|
339
339
|
}
|
|
340
340
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
+
import { atomicWriteFile } from "../state/atomic-write.ts";
|
|
3
4
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
4
5
|
import { projectCrewRoot } from "../utils/paths.ts";
|
|
5
6
|
import { assertSafePathId } from "../utils/safe-paths.ts";
|
|
@@ -73,22 +74,11 @@ export class FileCheckpointStore implements CheckpointStore {
|
|
|
73
74
|
// other's data (silent corruption) and the second rename hits ENOENT
|
|
74
75
|
// (silent data loss). Including taskId + pid + timestamp guarantees
|
|
75
76
|
// uniqueness across processes and across tasks.
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
fs.renameSync(tmp, p);
|
|
82
|
-
// fsync parent directory to ensure the rename is durable
|
|
83
|
-
const dirFd = fs.openSync(this.checkpointDir(), "r");
|
|
84
|
-
try {
|
|
85
|
-
fs.fsyncSync(dirFd);
|
|
86
|
-
} catch {
|
|
87
|
-
// EPERM on Windows: opening a directory and fsync-ing it is not supported
|
|
88
|
-
// on all Windows configurations. Best-effort — the rename is still atomic.
|
|
89
|
-
} finally {
|
|
90
|
-
fs.closeSync(dirFd);
|
|
91
|
-
}
|
|
77
|
+
// atomicWriteFile handles temp + write + fsync(data) + rename + fsync(parent)
|
|
78
|
+
// with a UUID-based unique temp name (cross-process safe). The previous
|
|
79
|
+
// hand-rolled version missed data-fsync (ST-5), risking empty/stale content
|
|
80
|
+
// after power loss.
|
|
81
|
+
atomicWriteFile(p, JSON.stringify(checkpoint, null, 2));
|
|
92
82
|
}
|
|
93
83
|
|
|
94
84
|
load(runId: string, taskId: string): Checkpoint | null {
|
package/src/runtime/child-pi.ts
CHANGED
|
@@ -5,6 +5,7 @@ import * as path from "node:path";
|
|
|
5
5
|
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
6
6
|
import { DEFAULT_CHILD_PI } from "../config/defaults.ts";
|
|
7
7
|
import { registerChildProcess, unregisterChildProcess } from "../extension/crew-cleanup.ts";
|
|
8
|
+
import { atomicWriteFile } from "../state/atomic-write.ts";
|
|
8
9
|
import type { WorkerExitStatus } from "../state/types.ts";
|
|
9
10
|
import { WINDOWS_ESSENTIAL_ENV_VARS } from "../utils/env-allowlist.ts";
|
|
10
11
|
import { buildScopedAllowList, sanitizeEnvSecrets } from "../utils/env-filter.ts";
|
|
@@ -31,6 +32,11 @@ const MAX_COMPACT_CONTENT_CHARS = DEFAULT_CHILD_PI.maxCompactContentChars;
|
|
|
31
32
|
const activeChildProcesses = new Map<number, ChildProcess>();
|
|
32
33
|
const childHardKillTimers = new Map<number, NodeJS.Timeout>();
|
|
33
34
|
|
|
35
|
+
/** Maximum size (bytes) for the ChildPiLineObserver's line accumulation buffer.
|
|
36
|
+
* When exceeded, the buffer is force-flushed to prevent unbounded memory growth
|
|
37
|
+
* from chatty child processes that produce output without newlines. */
|
|
38
|
+
const MAX_LINE_BUFFER_BYTES = 1024 * 1024; // 1 MB
|
|
39
|
+
|
|
34
40
|
// Periodic cleanup of dead child process entries to prevent memory leaks.
|
|
35
41
|
// If a child process never emits exit/close (zombie), the entry would leak.
|
|
36
42
|
setInterval(() => {
|
|
@@ -749,6 +755,20 @@ export class ChildPiLineObserver {
|
|
|
749
755
|
|
|
750
756
|
observe(text: string): void {
|
|
751
757
|
this.buffer += text;
|
|
758
|
+
// Cap the buffer to prevent unbounded memory growth when a child process
|
|
759
|
+
// produces output without newlines (RT-F8). When exceeded, force-flush
|
|
760
|
+
// the buffer as a single line and log a warning.
|
|
761
|
+
if (this.buffer.length > MAX_LINE_BUFFER_BYTES) {
|
|
762
|
+
logInternalError(
|
|
763
|
+
"child-pi.buffer-overflow",
|
|
764
|
+
new Error(`Line buffer exceeded ${MAX_LINE_BUFFER_BYTES} bytes; force-flushing`),
|
|
765
|
+
`bufferLen=${this.buffer.length}`,
|
|
766
|
+
);
|
|
767
|
+
const line = this.buffer;
|
|
768
|
+
this.buffer = "";
|
|
769
|
+
this.emitLine(line);
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
752
772
|
const lines = this.buffer.split(/\r?\n/);
|
|
753
773
|
this.buffer = lines.pop() ?? "";
|
|
754
774
|
for (const line of lines) this.emitLine(line);
|
|
@@ -869,7 +889,7 @@ function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
|
869
889
|
|
|
870
890
|
function isFinalAssistantEvent(event: unknown): boolean {
|
|
871
891
|
const obj = asRecord(event);
|
|
872
|
-
if (
|
|
892
|
+
if (obj?.type !== "message_end") return false;
|
|
873
893
|
const message = asRecord(obj.message);
|
|
874
894
|
const role = message?.role;
|
|
875
895
|
if (role !== undefined && role !== "assistant") return false;
|
|
@@ -1002,7 +1022,7 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
1002
1022
|
}
|
|
1003
1023
|
count += 1;
|
|
1004
1024
|
try {
|
|
1005
|
-
|
|
1025
|
+
atomicWriteFile(counterFile, String(count));
|
|
1006
1026
|
} catch (error) {
|
|
1007
1027
|
logInternalError("child-pi.mock-counter-write", error as Error, `file=${counterFile}`);
|
|
1008
1028
|
}
|
|
@@ -78,7 +78,7 @@ export function planCoalescedGroups(
|
|
|
78
78
|
const buckets = new Map<string, TeamTaskState[]>();
|
|
79
79
|
for (const taskId of readyTaskIds) {
|
|
80
80
|
const task = taskById.get(taskId);
|
|
81
|
-
if (!task
|
|
81
|
+
if (!task?.stepId) continue;
|
|
82
82
|
const step = stepById.get(task.stepId);
|
|
83
83
|
if (!step) continue;
|
|
84
84
|
|
|
@@ -397,31 +397,43 @@ export function purgeStaleActiveRunIndex(staleThresholdMs = 300_000, now = Date.
|
|
|
397
397
|
if (manifest?.status === "running" && manifest.async?.pid !== undefined) {
|
|
398
398
|
const pidAlive = checkProcessLiveness(manifest.async.pid).alive;
|
|
399
399
|
if (!pidAlive && !hasRecentLifeEvidence(entry, manifest.updatedAt, now, staleThresholdMs)) {
|
|
400
|
-
// Dead PID + no recent life evidence → cancel the manifest and unregister
|
|
400
|
+
// Dead PID + no recent life evidence → cancel the manifest and unregister.
|
|
401
|
+
// RT-F5: wrap load+modify+save in withRunLockSync so concurrent writers
|
|
402
|
+
// (e.g. team-runner final saveRunTasks) cannot interleave between the
|
|
403
|
+
// repaired-tasks write and the manifest-status flip. terminateLiveAgentsForRun
|
|
404
|
+
// is fire-and-forget so it stays OUTSIDE the lock to avoid blocking the
|
|
405
|
+
// critical section on async IPC.
|
|
401
406
|
try {
|
|
402
|
-
const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
|
|
407
|
+
const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
|
|
403
408
|
if (fullLoaded) {
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
409
|
+
withRunLockSync(fullLoaded.manifest, () => {
|
|
410
|
+
// Re-read under lock so the stale-best-effort load above
|
|
411
|
+
// can't cause us to clobber a fresh status flip.
|
|
412
|
+
const fresh = loadRunManifestById(entry.cwd, entry.runId);
|
|
413
|
+
if (!fresh) return;
|
|
414
|
+
if (fresh.manifest.status !== "running") return;
|
|
415
|
+
const now_iso = new Date(now).toISOString();
|
|
416
|
+
const repairedTasks = fresh.tasks.map((task) => {
|
|
417
|
+
if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
|
|
418
|
+
return {
|
|
419
|
+
...task,
|
|
420
|
+
status: "cancelled" as const,
|
|
421
|
+
finishedAt: now_iso,
|
|
422
|
+
error: "Orphaned run: worker process dead and no recent activity",
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
return task;
|
|
426
|
+
});
|
|
427
|
+
saveRunTasks(fresh.manifest, repairedTasks);
|
|
428
|
+
for (const task of repairedTasks) {
|
|
429
|
+
try {
|
|
430
|
+
upsertCrewAgent(fresh.manifest, recordFromTask(fresh.manifest, task, "scaffold"));
|
|
431
|
+
} catch {
|
|
432
|
+
/* non-critical */
|
|
433
|
+
}
|
|
413
434
|
}
|
|
414
|
-
|
|
435
|
+
updateRunStatus(fresh.manifest, "cancelled", "Orphaned run: worker process dead and no recent activity");
|
|
415
436
|
});
|
|
416
|
-
saveRunTasks(fullLoaded.manifest, repairedTasks);
|
|
417
|
-
for (const task of repairedTasks) {
|
|
418
|
-
try {
|
|
419
|
-
upsertCrewAgent(fullLoaded.manifest, recordFromTask(fullLoaded.manifest, task, "scaffold"));
|
|
420
|
-
} catch {
|
|
421
|
-
/* non-critical */
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
updateRunStatus(fullLoaded.manifest, "cancelled", "Orphaned run: worker process dead and no recent activity");
|
|
425
437
|
void terminateLiveAgentsForRun(
|
|
426
438
|
fullLoaded.manifest.runId,
|
|
427
439
|
"cancelled",
|
|
@@ -447,35 +459,39 @@ export function purgeStaleActiveRunIndex(staleThresholdMs = 300_000, now = Date.
|
|
|
447
459
|
if (manifest?.status === "running" && manifest.async === undefined) {
|
|
448
460
|
if (!hasRecentLifeEvidence(entry, manifest.updatedAt, now, staleThresholdMs)) {
|
|
449
461
|
try {
|
|
450
|
-
const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
|
|
462
|
+
const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
|
|
451
463
|
if (fullLoaded && fullLoaded.manifest.status === "running") {
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
if (
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
464
|
+
withRunLockSync(fullLoaded.manifest, () => {
|
|
465
|
+
const fresh = loadRunManifestById(entry.cwd, entry.runId);
|
|
466
|
+
if (fresh?.manifest.status !== "running") return;
|
|
467
|
+
const now_iso = new Date(now).toISOString();
|
|
468
|
+
const repairedTasks = fresh.tasks.map((task) => {
|
|
469
|
+
if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
|
|
470
|
+
return {
|
|
471
|
+
...task,
|
|
472
|
+
status: "cancelled" as const,
|
|
473
|
+
finishedAt: now_iso,
|
|
474
|
+
error: "Orphaned run: workflow completed but manifest never updated to terminal status",
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
return task;
|
|
478
|
+
});
|
|
479
|
+
saveRunTasks(fresh.manifest, repairedTasks);
|
|
480
|
+
for (const task of repairedTasks) {
|
|
481
|
+
try {
|
|
482
|
+
upsertCrewAgent(fresh.manifest, recordFromTask(fresh.manifest, task, "scaffold"));
|
|
483
|
+
} catch {
|
|
484
|
+
/* non-critical */
|
|
485
|
+
}
|
|
461
486
|
}
|
|
462
|
-
|
|
487
|
+
updateRunStatus(
|
|
488
|
+
fresh.manifest,
|
|
489
|
+
"cancelled",
|
|
490
|
+
"Orphaned run: no async worker and no manifest update in over " +
|
|
491
|
+
Math.round(staleThresholdMs / 60000) +
|
|
492
|
+
" minutes",
|
|
493
|
+
);
|
|
463
494
|
});
|
|
464
|
-
saveRunTasks(fullLoaded.manifest, repairedTasks);
|
|
465
|
-
for (const task of repairedTasks) {
|
|
466
|
-
try {
|
|
467
|
-
upsertCrewAgent(fullLoaded.manifest, recordFromTask(fullLoaded.manifest, task, "scaffold"));
|
|
468
|
-
} catch {
|
|
469
|
-
/* non-critical */
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
updateRunStatus(
|
|
473
|
-
fullLoaded.manifest,
|
|
474
|
-
"cancelled",
|
|
475
|
-
"Orphaned run: no async worker and no manifest update in over " +
|
|
476
|
-
Math.round(staleThresholdMs / 60000) +
|
|
477
|
-
" minutes",
|
|
478
|
-
);
|
|
479
495
|
void terminateLiveAgentsForRun(
|
|
480
496
|
fullLoaded.manifest.runId,
|
|
481
497
|
"cancelled",
|