pi-crew 0.8.13 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +296 -0
- package/README.md +118 -2
- package/docs/FEATURE_INTAKE.md +1 -1
- package/docs/HARNESS.md +20 -19
- package/docs/PROJECT_REVIEW.md +132 -133
- package/docs/PROJECT_REVIEW_FIXES.md +130 -131
- package/docs/actions-reference.md +127 -121
- package/docs/architecture.md +1 -1
- package/docs/code-review-2026-05-11.md +134 -134
- package/docs/commands-reference.md +108 -106
- package/docs/comparison-pi-subagents-vs-pi-crew.md +105 -105
- package/docs/deep-review-report.md +1 -1
- package/docs/dynamic-workflows.md +90 -0
- package/docs/fixes/BATCH_A_H1_H2.md +17 -17
- package/docs/fixes/bug-007-async-notifier-stale-ctx.md +23 -23
- package/docs/followup-plan-2026-05-12.md +135 -135
- package/docs/followup-review-2026-05-12.md +86 -86
- package/docs/followup-review-round3-2026-05-12.md +123 -123
- package/docs/goals.md +59 -0
- package/docs/implementation-plan-top3.md +4 -4
- package/docs/issue-29-analysis.md +2 -2
- package/docs/oh-my-pi-research.md +154 -154
- package/docs/optimization-plan.md +2 -0
- package/docs/perf/baseline-2026-05.md +9 -9
- package/docs/perf/final-report-2026-05.md +2 -2
- package/docs/perf/sprint-1-report.md +2 -2
- package/docs/perf/sprint-2-report.md +1 -1
- package/docs/perf/upgrade-plan-2026-05.md +72 -72
- package/docs/pi-crew-bugs.md +230 -230
- package/docs/pi-crew-investigation-report.md +102 -102
- package/docs/pi-crew-test-round5.md +4 -4
- package/docs/runtime-analysis-child-vs-live.md +57 -57
- package/docs/runtime-migration-in-process-analysis.md +97 -97
- package/install.mjs +3 -2
- package/package.json +2 -4
- package/skills/orchestration/SKILL.md +11 -11
- package/src/agents/agent-config.ts +4 -0
- package/src/config/config.ts +39 -0
- package/src/config/types.ts +11 -0
- package/src/extension/action-suggestions.ts +2 -1
- package/src/extension/async-notifier.ts +10 -0
- package/src/extension/help.ts +14 -0
- package/src/extension/project-init.ts +7 -20
- package/src/extension/registration/commands.ts +27 -0
- package/src/extension/team-tool/destructive-gate.ts +1 -1
- package/src/extension/team-tool/goal-wrap.ts +288 -0
- package/src/extension/team-tool/goal.ts +405 -0
- package/src/extension/team-tool/run.ts +103 -4
- package/src/extension/team-tool/workflow-manage.ts +194 -0
- package/src/extension/team-tool.ts +20 -0
- package/src/hooks/types.ts +3 -1
- package/src/runtime/async-runner.ts +24 -2
- package/src/runtime/background-runner.ts +68 -19
- package/src/runtime/child-pi.ts +6 -1
- package/src/runtime/completion-guard.ts +1 -1
- package/src/runtime/dynamic-workflow-context.ts +450 -0
- package/src/runtime/dynamic-workflow-runner.ts +180 -0
- package/src/runtime/global-worker-cap.ts +96 -0
- package/src/runtime/goal-evaluator.ts +294 -0
- package/src/runtime/goal-loop-runner.ts +612 -0
- package/src/runtime/goal-state-store.ts +209 -0
- package/src/runtime/pi-args.ts +10 -2
- package/src/runtime/result-extractor.ts +32 -0
- package/src/runtime/team-runner.ts +11 -1
- package/src/runtime/verification-gates.ts +85 -5
- package/src/runtime/verification-integrity.ts +110 -0
- package/src/runtime/verification-worktree.ts +136 -0
- package/src/runtime/workspace-lock.ts +448 -0
- package/src/schema/config-schema.ts +26 -0
- package/src/schema/team-tool-schema.ts +39 -4
- package/src/state/atomic-write.ts +9 -0
- package/src/state/contracts.ts +14 -0
- package/src/state/crew-init.ts +18 -5
- package/src/state/event-log.ts +7 -1
- package/src/state/state-store.ts +2 -0
- package/src/state/types.ts +82 -0
- package/src/state/worker-atomic-writer.ts +176 -0
- package/src/utils/redaction.ts +104 -24
- package/src/workflows/discover-workflows.ts +25 -1
- package/src/workflows/workflow-config.ts +13 -0
- package/teams/parallel-research.team.md +1 -1
- package/workflows/examples/hello.dwf.ts +24 -0
package/src/state/state-store.ts
CHANGED
|
@@ -227,6 +227,7 @@ export function createRunManifest(params: {
|
|
|
227
227
|
goal: string;
|
|
228
228
|
workspaceMode?: "single" | "worktree";
|
|
229
229
|
ownerSessionId?: string;
|
|
230
|
+
runKind?: "team-run" | "goal-loop" | "dynamic-workflow";
|
|
230
231
|
}): { manifest: TeamRunManifest; tasks: TeamTaskState[]; paths: RunPaths } {
|
|
231
232
|
const paths = createRunPaths(params.cwd);
|
|
232
233
|
const now = new Date().toISOString();
|
|
@@ -249,6 +250,7 @@ export function createRunManifest(params: {
|
|
|
249
250
|
eventsPath: paths.eventsPath,
|
|
250
251
|
artifacts: [],
|
|
251
252
|
...(params.ownerSessionId ? { ownerSessionId: params.ownerSessionId } : {}),
|
|
253
|
+
runKind: params.runKind ?? "team-run",
|
|
252
254
|
};
|
|
253
255
|
fs.mkdirSync(paths.stateRoot, { recursive: true });
|
|
254
256
|
fs.mkdirSync(paths.artifactsRoot, { recursive: true });
|
package/src/state/types.ts
CHANGED
|
@@ -183,6 +183,8 @@ export interface TeamRunManifest {
|
|
|
183
183
|
runtimeResolution?: RuntimeResolutionState;
|
|
184
184
|
/** Effective run config snapshot used by async background workers. Optional for backward compatibility. */
|
|
185
185
|
runConfig?: unknown;
|
|
186
|
+
/** Background dispatch discriminator. Default "team-run" runs executeTeamRun; "goal-loop" / "dynamic-workflow" dispatch to their respective runners. Absent = "team-run" for backward compatibility. */
|
|
187
|
+
runKind?: "team-run" | "goal-loop" | "dynamic-workflow";
|
|
186
188
|
summary?: string;
|
|
187
189
|
policyDecisions?: PolicyDecision[];
|
|
188
190
|
}
|
|
@@ -196,6 +198,86 @@ export interface UsageState {
|
|
|
196
198
|
turns?: number;
|
|
197
199
|
}
|
|
198
200
|
|
|
201
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
202
|
+
// Goal loop types (P0/P1 — autonomous goal loop, Claude-Code-style /goal).
|
|
203
|
+
// Spec: research-findings/goal-workflow/00-SPEC.md §2.3; plan 07-PLAN.md v3 §0b G2 + §0c.
|
|
204
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Outer-state lifecycle of a goal loop. Inner per-turn state lives on each turn's TeamRunManifest.
|
|
208
|
+
*
|
|
209
|
+
* P1b (RFC v0.5 §P1b): `"stuck"` is NON-TERMINAL and RE-HINTABLE. Legal transitions:
|
|
210
|
+
* running → stuck (only by the background loop, after the oscillation detector fires)
|
|
211
|
+
* stuck → running (only by `goal resume`, atomically via GoalStore.compareAndSetStatus)
|
|
212
|
+
* stuck → cancelled (by the idle-timeout sweeper OR `goal stop`)
|
|
213
|
+
*/
|
|
214
|
+
export type GoalLoopStatus =
|
|
215
|
+
| "running"
|
|
216
|
+
| "paused"
|
|
217
|
+
| "stuck"
|
|
218
|
+
| "achieved"
|
|
219
|
+
| "max_turns"
|
|
220
|
+
| "budget_exceeded"
|
|
221
|
+
| "blocked"
|
|
222
|
+
| "cancelled";
|
|
223
|
+
|
|
224
|
+
/** One evaluation by the goal-judge model after a turn. */
|
|
225
|
+
export interface GoalVerdict {
|
|
226
|
+
turn: number;
|
|
227
|
+
achieved: boolean;
|
|
228
|
+
/** "achieved: all tests pass" | "not-achieved: 2/8 tests failing" | "BLOCKED: <reason>" (BLOCKED: prefix → status='blocked'). */
|
|
229
|
+
reason: string;
|
|
230
|
+
evidenceRefs?: string[];
|
|
231
|
+
evaluatorModel: string;
|
|
232
|
+
evaluatedAt: string;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Persisted at <crewRoot>/state/goals/<goalId>.json by GoalStore. Survives session restart. */
|
|
236
|
+
export interface GoalLoopState {
|
|
237
|
+
goalId: string;
|
|
238
|
+
ownerSessionId: string;
|
|
239
|
+
objective: string;
|
|
240
|
+
scope?: string;
|
|
241
|
+
/** Acceptance conditions as shell commands (exit 0 = pass). Reuses VerificationContract semantics. */
|
|
242
|
+
verification?: { commands: string[]; allowManualEvidence?: boolean };
|
|
243
|
+
state: GoalLoopStatus;
|
|
244
|
+
maxTurns: number;
|
|
245
|
+
turnsUsed: number;
|
|
246
|
+
budgetTotal?: number;
|
|
247
|
+
/** P1d (RFC v0.5 §P1d): when true, budget enforcement is skipped (explicit opt-out; audit-logged at start). */
|
|
248
|
+
budgetUnlimited?: boolean;
|
|
249
|
+
budgetWarning?: number;
|
|
250
|
+
budgetAbort?: number;
|
|
251
|
+
budgetUsed: number;
|
|
252
|
+
/**
|
|
253
|
+
* P1a (RFC v0.5 §P1a): bookend integrity snapshot of project-manifest files
|
|
254
|
+
* taken at goal start (only when verification.commands is declared). The
|
|
255
|
+
* goal-loop-runner re-hashes before (T_snap) and after (T_verify_done) each
|
|
256
|
+
* verification command to detect persistent manifest tampering. The literal
|
|
257
|
+
* `"none-text-only"` marks goals started in text-only verification mode
|
|
258
|
+
* (no objective oracle → no snapshot taken).
|
|
259
|
+
*/
|
|
260
|
+
verificationIntegrity?:
|
|
261
|
+
| { snapshot: Record<string, string>; takenAt: string }
|
|
262
|
+
| "none-text-only";
|
|
263
|
+
evaluatorModel: string;
|
|
264
|
+
workerModel?: string;
|
|
265
|
+
/** subagent_type / agent name for worker turns (default "executor"). */
|
|
266
|
+
workerAgent?: string;
|
|
267
|
+
team?: string;
|
|
268
|
+
cwd: string;
|
|
269
|
+
/** Feedback from turn N's verdict, prepended into turn N+1's manifest.goal (G1). */
|
|
270
|
+
nextTurnFeedback?: string;
|
|
271
|
+
/** The team-run of the current in-flight turn (for cancel/steer). */
|
|
272
|
+
currentRunId?: string;
|
|
273
|
+
verdicts: GoalVerdict[];
|
|
274
|
+
history: { runId: string; outcome: string; learnedAt: string; turn: number }[];
|
|
275
|
+
createdAt: string;
|
|
276
|
+
updatedAt: string;
|
|
277
|
+
/** Mirror of manifest.async for PID-liveness checks (cf. AsyncRunState). */
|
|
278
|
+
async?: { pid: number; logPath: string; spawnedAt: string };
|
|
279
|
+
}
|
|
280
|
+
|
|
199
281
|
export interface ModelAttemptState {
|
|
200
282
|
model: string;
|
|
201
283
|
success: boolean;
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1.5 worker-thread atomic writer (RFC: 15-PHASE1.5-WORKER-WRITER-RFC.md).
|
|
3
|
+
*
|
|
4
|
+
* Background: multi-step goal-wrapped workflows crash silently and
|
|
5
|
+
* non-deterministically during batch transitions. The crash point moves to
|
|
6
|
+
* every `await` yield in the write path (mkdir, open, stat, rename,
|
|
7
|
+
* appendEvent). Sync replacements regress. Hypothesis: V8/libuv-level race
|
|
8
|
+
* during event-loop yields. Mitigation: route writes through a dedicated
|
|
9
|
+
* worker thread that performs SYNC fs operations with no internal yields.
|
|
10
|
+
*
|
|
11
|
+
* Opt-in via `PI_CREW_WORKER_ATOMIC_WRITER=1`. When disabled, callers fall
|
|
12
|
+
* back to the regular async path. Safe to ship behind a flag.
|
|
13
|
+
*
|
|
14
|
+
* Protocol (main → worker):
|
|
15
|
+
* { kind: "write", id, filePath, content }
|
|
16
|
+
* { kind: "mkdir", id, dirPath }
|
|
17
|
+
* { kind: "append", id, filePath, content }
|
|
18
|
+
*
|
|
19
|
+
* Protocol (worker → main):
|
|
20
|
+
* { kind: "done", id }
|
|
21
|
+
* { kind: "error", id, message }
|
|
22
|
+
*/
|
|
23
|
+
import { Worker } from "node:worker_threads";
|
|
24
|
+
import * as path from "node:path";
|
|
25
|
+
import * as fs from "node:fs";
|
|
26
|
+
import { createRequire } from "node:module";
|
|
27
|
+
|
|
28
|
+
const require = createRequire(import.meta.url);
|
|
29
|
+
|
|
30
|
+
let worker: Worker | undefined;
|
|
31
|
+
let nextRequestId = 1;
|
|
32
|
+
const pending = new Map<number, { resolve: () => void; reject: (e: Error) => void }>();
|
|
33
|
+
|
|
34
|
+
/** Worker script source — runs SYNC fs ops with no internal yields. */
|
|
35
|
+
const WORKER_SOURCE = `
|
|
36
|
+
const { parentPort, workerData } = require("node:worker_threads");
|
|
37
|
+
const fs = require("node:fs");
|
|
38
|
+
const path = require("node:path");
|
|
39
|
+
const crypto = require("node:crypto");
|
|
40
|
+
|
|
41
|
+
function isSymlinkSafePath(filePath) {
|
|
42
|
+
try {
|
|
43
|
+
let currentPath = filePath;
|
|
44
|
+
while (currentPath !== path.dirname(currentPath)) {
|
|
45
|
+
const dir = path.dirname(currentPath);
|
|
46
|
+
try {
|
|
47
|
+
const stat = fs.lstatSync(dir);
|
|
48
|
+
if (stat.isSymbolicLink()) {
|
|
49
|
+
// Accept symlinks under /tmp (macOS /var/folders) and project dirs;
|
|
50
|
+
// reject others. Mirrors atomic-write.ts policy for goal-loop paths.
|
|
51
|
+
const real = fs.realpathSync(dir);
|
|
52
|
+
if (!real.startsWith("/tmp/") && !real.startsWith(process.cwd())) {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
} catch { /* not found — fine */ }
|
|
57
|
+
currentPath = dir;
|
|
58
|
+
}
|
|
59
|
+
return true;
|
|
60
|
+
} catch { return true; }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function syncAtomicWriteFile(filePath, content) {
|
|
64
|
+
if (!isSymlinkSafePath(filePath)) throw new Error("Refusing to write: unsafe path: " + filePath);
|
|
65
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
66
|
+
const tempPath = filePath + "." + crypto.randomUUID() + ".tmp";
|
|
67
|
+
try {
|
|
68
|
+
const fd = fs.openSync(tempPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600);
|
|
69
|
+
try {
|
|
70
|
+
fs.writeFileSync(fd, content, "utf-8");
|
|
71
|
+
} finally {
|
|
72
|
+
fs.closeSync(fd);
|
|
73
|
+
}
|
|
74
|
+
fs.renameSync(tempPath, filePath);
|
|
75
|
+
} catch (error) {
|
|
76
|
+
try { fs.rmSync(tempPath, { force: true }); } catch {}
|
|
77
|
+
// If rename raced with another writer that produced identical content, swallow.
|
|
78
|
+
if (error && error.code === "EEXIST") {
|
|
79
|
+
try {
|
|
80
|
+
const existing = fs.readFileSync(filePath, "utf-8");
|
|
81
|
+
if (existing === content) return;
|
|
82
|
+
} catch {}
|
|
83
|
+
}
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function syncAppend(filePath, content) {
|
|
89
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
90
|
+
fs.appendFileSync(filePath, content, "utf-8");
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
parentPort.on("message", (msg) => {
|
|
94
|
+
try {
|
|
95
|
+
if (msg.kind === "write") syncAtomicWriteFile(msg.filePath, msg.content);
|
|
96
|
+
else if (msg.kind === "mkdir") fs.mkdirSync(msg.dirPath, { recursive: true });
|
|
97
|
+
else if (msg.kind === "append") syncAppend(msg.filePath, msg.content);
|
|
98
|
+
else throw new Error("worker-atomic-writer: unknown message kind: " + msg.kind);
|
|
99
|
+
parentPort.postMessage({ kind: "done", id: msg.id });
|
|
100
|
+
} catch (error) {
|
|
101
|
+
parentPort.postMessage({ kind: "error", id: msg.id, message: error && error.message ? error.message : String(error) });
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
`;
|
|
105
|
+
|
|
106
|
+
function getWorker(): Worker {
|
|
107
|
+
if (worker) return worker;
|
|
108
|
+
// Write worker source to a temp file (so Worker can load it as CJS via
|
|
109
|
+
// require() inside the worker, which always has CommonJS available).
|
|
110
|
+
const os = require("node:os");
|
|
111
|
+
const tmpPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "pi-crew-waw-")), "worker.cjs");
|
|
112
|
+
fs.writeFileSync(tmpPath, WORKER_SOURCE, "utf-8");
|
|
113
|
+
worker = new Worker(tmpPath);
|
|
114
|
+
worker.on("message", (msg: { kind: string; id: number; message?: string }) => {
|
|
115
|
+
const entry = pending.get(msg.id);
|
|
116
|
+
if (!entry) return;
|
|
117
|
+
pending.delete(msg.id);
|
|
118
|
+
if (msg.kind === "done") entry.resolve();
|
|
119
|
+
else entry.reject(new Error(msg.message ?? "worker-atomic-writer: unknown error"));
|
|
120
|
+
});
|
|
121
|
+
worker.on("error", (error: Error) => {
|
|
122
|
+
// Reject ALL pending requests — worker died.
|
|
123
|
+
for (const [, entry] of pending) entry.reject(error);
|
|
124
|
+
pending.clear();
|
|
125
|
+
});
|
|
126
|
+
worker.unref(); // don't keep event loop alive (unless tests request it)
|
|
127
|
+
if (keepRefForTests) worker.ref();
|
|
128
|
+
return worker;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function dispatch(kind: "write" | "mkdir" | "append", payload: Record<string, unknown>): Promise<void> {
|
|
132
|
+
return new Promise((resolve, reject) => {
|
|
133
|
+
const id = nextRequestId++;
|
|
134
|
+
pending.set(id, { resolve, reject });
|
|
135
|
+
try {
|
|
136
|
+
getWorker().postMessage({ kind, id, ...payload });
|
|
137
|
+
} catch (error) {
|
|
138
|
+
pending.delete(id);
|
|
139
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Whether the worker writer is enabled (env var opt-in). */
|
|
145
|
+
export function isWorkerAtomicWriterEnabled(): boolean {
|
|
146
|
+
return process.env.PI_CREW_WORKER_ATOMIC_WRITER === "1" || process.env.PI_TEAMS_WORKER_ATOMIC_WRITER === "1";
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Atomic-write a file via the worker thread. Sync fs ops inside worker. */
|
|
150
|
+
export function atomicWriteFileViaWorker(filePath: string, content: string): Promise<void> {
|
|
151
|
+
return dispatch("write", { filePath, content });
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Append to a file via the worker thread (used by event-log). */
|
|
155
|
+
export function appendFileViaWorker(filePath: string, content: string): Promise<void> {
|
|
156
|
+
return dispatch("append", { filePath, content });
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Terminate the worker (for tests / cleanup). */
|
|
160
|
+
export function terminateWorkerAtomicWriter(): void {
|
|
161
|
+
if (worker) {
|
|
162
|
+
const w = worker;
|
|
163
|
+
worker = undefined;
|
|
164
|
+
w.terminate().catch(() => { /* ignore */ });
|
|
165
|
+
}
|
|
166
|
+
for (const [, entry] of pending) entry.reject(new Error("worker terminated"));
|
|
167
|
+
pending.clear();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Tests-only knob: keep the worker ref'd so the test runner doesn't exit
|
|
171
|
+
* before promises resolve. Production code leaves this false (worker is
|
|
172
|
+
* unref'd to avoid blocking process exit). */
|
|
173
|
+
let keepRefForTests = false;
|
|
174
|
+
export function __setKeepWorkerRefForTests(value: boolean): void {
|
|
175
|
+
keepRefForTests = value;
|
|
176
|
+
}
|
package/src/utils/redaction.ts
CHANGED
|
@@ -6,6 +6,36 @@
|
|
|
6
6
|
// Pattern for PEM private keys (possessive quantifier prevents backtracking)
|
|
7
7
|
export const PEM_PRIVATE_KEY_PATTERN = /-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]+?-----END [A-Z ]+PRIVATE KEY-----/g;
|
|
8
8
|
|
|
9
|
+
// --- P1f (RFC §P1f / §6 STRIDE) — additional anchored, ReDoS-SAFE secret patterns. ---
|
|
10
|
+
// All patterns below are LINEAR-TIME: each uses a single bounded quantifier on a
|
|
11
|
+
// character class (fixed {N} or a plain +) with NO nested quantifiers and NO
|
|
12
|
+
// overlapping alternation. Boundaries are zero-width lookarounds on simple char
|
|
13
|
+
// classes, which are also linear. Do NOT introduce (a+)+-style nesting here.
|
|
14
|
+
//
|
|
15
|
+
// RESIDUAL (documented, Med-High per RFC §6): regex redaction is BEST-EFFORT
|
|
16
|
+
// against an *adversarial* worker that can encode/split/transform secrets
|
|
17
|
+
// (base64, line splits, novel formats, non-pattern env vars). This catches the
|
|
18
|
+
// common/accidental leak; it is NOT a boundary against a determined exfiltrator.
|
|
19
|
+
// Full mitigation ladder: (1) redaction here + at artifact-write; (2) Phase 1.5
|
|
20
|
+
// sanitized-env verification; (3) sandbox (deferred).
|
|
21
|
+
|
|
22
|
+
// JWT — three base64url segments separated by dots, distinctive "eyJ" headers.
|
|
23
|
+
// Linear: single + on [A-Za-z0-9_-] per segment, no nesting.
|
|
24
|
+
export const JWT_PATTERN = /(?<![A-Za-z0-9_-])eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
|
|
25
|
+
|
|
26
|
+
// GitHub PAT (classic + fine-grained prefixes) — fixed 36-char base62 tail.
|
|
27
|
+
// Linear: fixed {36} count on a char class (constant time per match position).
|
|
28
|
+
export const GITHUB_PAT_PATTERN = /(?<![A-Za-z0-9_])gh[pousr]_[A-Za-z0-9]{36}(?![A-Za-z0-9])/g;
|
|
29
|
+
|
|
30
|
+
// AWS access key id — fixed 16-char uppercase-alphanumeric tail.
|
|
31
|
+
// Linear: fixed {16} count on a char class.
|
|
32
|
+
export const AWS_ACCESS_KEY_PATTERN = /(?<![A-Za-z0-9])AKIA[0-9A-Z]{16}(?![0-9A-Z])/g;
|
|
33
|
+
|
|
34
|
+
// Optional extras (RFC OQ13) — same ReDoS-safe shape (fixed counts / single +).
|
|
35
|
+
export const SLACK_TOKEN_PATTERN = /(?<![A-Za-z0-9_-])xox[baprs]-[A-Za-z0-9-]{10,}/g;
|
|
36
|
+
export const GOOGLE_API_KEY_PATTERN = /(?<![A-Za-z0-9_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g;
|
|
37
|
+
export const STRIPE_KEY_PATTERN = /(?<![A-Za-z0-9_])sk_live_[0-9a-zA-Z]{24}(?![0-9a-zA-Z])/g;
|
|
38
|
+
|
|
9
39
|
// Linear-time secret key detection
|
|
10
40
|
// IMPORTANT: This function must maintain linear-time guarantees.
|
|
11
41
|
// The fast-path regex uses simple string alternatives with anchors only (no quantifiers),
|
|
@@ -18,23 +48,52 @@ export function isSecretKey(keyName: string): boolean {
|
|
|
18
48
|
if (/^(token|apikey|api_key|password|secret|credential|authorization|privatekey|private_key)$/.test(lower)) {
|
|
19
49
|
return true;
|
|
20
50
|
}
|
|
21
|
-
// Linear scan for prefix characters followed by keywords
|
|
51
|
+
// Linear scan for prefix characters followed by keywords.
|
|
52
|
+
// FIX (cold-review #1 of Phase 1): use `lower.startsWith(kw, i+1)` + `lower.charAt(...)` to
|
|
53
|
+
// avoid allocating substring+toLowerCase inside the O(n) loop (was O(n^2) — adversarial
|
|
54
|
+
// worker emitting `_`×N+`=` stalled 200KB→29s, 500KB→216s). Now O(n).
|
|
22
55
|
const prefixes = "_.-";
|
|
23
56
|
const keywords = ["token", "api", "key", "password", "passwd", "secret", "credential", "authorization", "private"];
|
|
24
|
-
|
|
57
|
+
|
|
25
58
|
for (let i = 0; i < keyName.length; i++) {
|
|
26
59
|
if (prefixes.includes(keyName[i])) {
|
|
27
|
-
const remaining = keyName.substring(i + 1).toLowerCase();
|
|
28
60
|
for (const kw of keywords) {
|
|
29
|
-
if (
|
|
30
|
-
const
|
|
31
|
-
if (
|
|
61
|
+
if (lower.startsWith(kw, i + 1)) {
|
|
62
|
+
const afterCh = lower.charAt(i + 1 + kw.length);
|
|
63
|
+
if (afterCh === "" || prefixes.includes(afterCh) || /[a-zA-Z0-9]/.test(afterCh)) {
|
|
32
64
|
return true;
|
|
33
65
|
}
|
|
34
66
|
}
|
|
35
67
|
}
|
|
36
68
|
}
|
|
37
69
|
}
|
|
70
|
+
// FIX (P1f, surfaced by notification-sink test): also match camelCase
|
|
71
|
+
// boundaries (e.g. `apiToken`, `clientSecret`, `authKey`) — the separator
|
|
72
|
+
// scan above requires `_-.` between prefix and keyword and MISSES the very
|
|
73
|
+
// common camelCase pattern. Scan: a keyword matches if it appears with a
|
|
74
|
+
// word boundary (start of string, end of string, camelCase lowercase->upper
|
|
75
|
+
// transition, or one of `_-.` separators). Linear: one forward pass.
|
|
76
|
+
for (const kw of keywords) {
|
|
77
|
+
let from = 0;
|
|
78
|
+
while (true) {
|
|
79
|
+
const idx = lower.indexOf(kw, from);
|
|
80
|
+
if (idx === -1) break;
|
|
81
|
+
const before = idx === 0 ? "" : lower.charAt(idx - 1);
|
|
82
|
+
const afterIdx = idx + kw.length;
|
|
83
|
+
const afterCh = afterIdx >= lower.length ? "" : lower.charAt(afterIdx);
|
|
84
|
+
const atStart = idx === 0;
|
|
85
|
+
const atEnd = afterIdx === lower.length;
|
|
86
|
+
const camelBoundary = /[A-Z]/.test(keyName.charAt(afterIdx)); // lowercase->uppercase in original
|
|
87
|
+
const sepBoundary = prefixes.includes(before) || prefixes.includes(afterCh);
|
|
88
|
+
if (atStart || atEnd || camelBoundary || sepBoundary) {
|
|
89
|
+
// Require non-empty chars before/after to avoid matching `api` inside `capitalize`
|
|
90
|
+
const hasBefore = idx > 0;
|
|
91
|
+
const hasAfter = afterIdx < lower.length;
|
|
92
|
+
if (hasBefore || hasAfter) return true;
|
|
93
|
+
}
|
|
94
|
+
from = idx + 1;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
38
97
|
return false;
|
|
39
98
|
}
|
|
40
99
|
|
|
@@ -124,9 +183,20 @@ export function redactSecretString(value: string): string {
|
|
|
124
183
|
// Replace Authorization headers (non-Bearer format)
|
|
125
184
|
result = redactAuthHeader(result);
|
|
126
185
|
|
|
127
|
-
// Replace Bearer tokens
|
|
186
|
+
// Replace Bearer tokens (run before structured-token patterns so a
|
|
187
|
+
// "Bearer <jwt>" pair is collapsed first; bare tokens are caught below).
|
|
128
188
|
result = redactBearerTokens(result);
|
|
129
189
|
|
|
190
|
+
// P1f: structured secret tokens (JWT / GitHub PAT / AWS keys + optional
|
|
191
|
+
// Slack/Google/Stripe). Best-effort vs adversarial workers (see note above).
|
|
192
|
+
result = result
|
|
193
|
+
.replace(JWT_PATTERN, "***")
|
|
194
|
+
.replace(GITHUB_PAT_PATTERN, "***")
|
|
195
|
+
.replace(AWS_ACCESS_KEY_PATTERN, "***")
|
|
196
|
+
.replace(SLACK_TOKEN_PATTERN, "***")
|
|
197
|
+
.replace(GOOGLE_API_KEY_PATTERN, "***")
|
|
198
|
+
.replace(STRIPE_KEY_PATTERN, "***");
|
|
199
|
+
|
|
130
200
|
// Replace inline secrets: key=value or key:value patterns
|
|
131
201
|
result = redactInlineSecrets(result);
|
|
132
202
|
|
|
@@ -134,51 +204,61 @@ export function redactSecretString(value: string): string {
|
|
|
134
204
|
}
|
|
135
205
|
|
|
136
206
|
// Linear-time inline secret redaction: token=xxx, api_key=xxx, etc.
|
|
207
|
+
// FIX (P1f): previously O(n^2) — after a non-secret alphanumeric run, the loop did
|
|
208
|
+
// i++ (advance 1 char) and re-scanned from i+1, so a long run was rescanned O(n)
|
|
209
|
+
// times = O(n^2). The P1f ReDoS test (300KB no-dot input) surfaced this pre-existing
|
|
210
|
+
// bug. Now advances past the whole run when it isn't a redactable secret -> O(n).
|
|
137
211
|
function redactInlineSecrets(value: string): string {
|
|
138
212
|
const result: string[] = [];
|
|
139
213
|
let i = 0;
|
|
140
|
-
|
|
214
|
+
|
|
141
215
|
while (i < value.length) {
|
|
142
|
-
//
|
|
143
|
-
// Check for secret key followed by = or :
|
|
216
|
+
// Collect a run of key characters (alphanumeric, underscore, hyphen).
|
|
144
217
|
let j = i;
|
|
145
|
-
let keyLen = 0;
|
|
146
|
-
|
|
147
|
-
// Collect key characters (alphanumeric, underscore, hyphen)
|
|
148
218
|
while (j < value.length && /[a-zA-Z0-9_-]/.test(value[j])) {
|
|
149
219
|
j++;
|
|
150
|
-
keyLen++;
|
|
151
220
|
}
|
|
152
|
-
|
|
221
|
+
const keyLen = j - i;
|
|
222
|
+
|
|
223
|
+
let redacted = false;
|
|
153
224
|
if (keyLen > 0 && j < value.length && (value[j] === '=' || value[j] === ':')) {
|
|
154
|
-
const key = value.substring(i,
|
|
155
|
-
|
|
225
|
+
const key = value.substring(i, j);
|
|
226
|
+
|
|
156
227
|
// Check if this is a secret key
|
|
157
228
|
if (isSecretKey(key)) {
|
|
158
229
|
// Find the value (everything after = or : until space, comma, or end)
|
|
159
230
|
const sep = value[j];
|
|
160
231
|
let k = j + 1;
|
|
161
232
|
let valLen = 0;
|
|
162
|
-
while (k < value.length && valLen < 500 && value[k] !== ' ' && value[k] !== ',' && value[k] !== ';' && value[k] !== '"' && value[k] !== '
|
|
233
|
+
while (k < value.length && valLen < 500 && value[k] !== ' ' && value[k] !== ',' && value[k] !== ';' && value[k] !== '"' && value[k] !== '\r' && value[k] !== '\n') {
|
|
163
234
|
k++;
|
|
164
235
|
valLen++;
|
|
165
236
|
}
|
|
166
|
-
|
|
237
|
+
|
|
167
238
|
// Only redact if there's actual content
|
|
168
239
|
if (valLen > 0) {
|
|
169
240
|
result.push(key);
|
|
170
241
|
result.push(sep);
|
|
171
242
|
result.push("***");
|
|
172
243
|
i = k;
|
|
173
|
-
|
|
244
|
+
redacted = true;
|
|
174
245
|
}
|
|
175
246
|
}
|
|
176
247
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
248
|
+
|
|
249
|
+
if (!redacted) {
|
|
250
|
+
if (keyLen > 0) {
|
|
251
|
+
// Not a redactable secret — push the WHOLE run and advance past it (O(n)).
|
|
252
|
+
result.push(value.substring(i, j));
|
|
253
|
+
i = j;
|
|
254
|
+
} else {
|
|
255
|
+
// Single non-key character (space, punctuation, etc.)
|
|
256
|
+
result.push(value[i]);
|
|
257
|
+
i++;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
180
260
|
}
|
|
181
|
-
|
|
261
|
+
|
|
182
262
|
return result.join("");
|
|
183
263
|
}
|
|
184
264
|
|
|
@@ -120,11 +120,35 @@ function parseWorkflowFile(filePath: string, source: ResourceSource): WorkflowCo
|
|
|
120
120
|
|
|
121
121
|
function readWorkflowDir(dir: string, source: ResourceSource): WorkflowConfig[] {
|
|
122
122
|
if (!fs.existsSync(dir)) return [];
|
|
123
|
-
|
|
123
|
+
const staticWorkflows = fs.readdirSync(dir)
|
|
124
124
|
.filter((entry) => entry.endsWith(".workflow.md"))
|
|
125
125
|
.map((entry) => parseWorkflowFile(path.join(dir, entry), source))
|
|
126
126
|
.filter((workflow): workflow is WorkflowConfig => workflow !== undefined)
|
|
127
127
|
.sort((a, b) => a.name.localeCompare(b.name));
|
|
128
|
+
// P2: also discover dynamic workflows (*.dwf.ts). A .dwf.ts's default export is a JS orchestrator.
|
|
129
|
+
const dynamicWorkflows = fs.readdirSync(dir)
|
|
130
|
+
.filter((entry) => entry.endsWith(".dwf.ts"))
|
|
131
|
+
.map((entry) => parseDynamicWorkflowFile(path.join(dir, entry), source))
|
|
132
|
+
.filter((workflow): workflow is WorkflowConfig => workflow !== undefined);
|
|
133
|
+
return [...staticWorkflows, ...dynamicWorkflows].sort((a, b) => a.name.localeCompare(b.name));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** P2: a .dwf.ts is a dynamic workflow. Name = filename stem; script = the file itself. */
|
|
137
|
+
function parseDynamicWorkflowFile(filePath: string, source: ResourceSource): WorkflowConfig | undefined {
|
|
138
|
+
try {
|
|
139
|
+
const basename = path.basename(filePath, ".dwf.ts");
|
|
140
|
+
return {
|
|
141
|
+
name: basename,
|
|
142
|
+
description: `Dynamic workflow script (${basename}.dwf.ts).`,
|
|
143
|
+
source,
|
|
144
|
+
filePath,
|
|
145
|
+
steps: [],
|
|
146
|
+
runtime: "dynamic",
|
|
147
|
+
dynamicScript: filePath,
|
|
148
|
+
};
|
|
149
|
+
} catch {
|
|
150
|
+
return undefined;
|
|
151
|
+
}
|
|
128
152
|
}
|
|
129
153
|
|
|
130
154
|
export function discoverWorkflows(cwd: string): WorkflowDiscoveryResult {
|
|
@@ -39,4 +39,17 @@ export interface WorkflowConfig {
|
|
|
39
39
|
filePath: string;
|
|
40
40
|
steps: WorkflowStep[];
|
|
41
41
|
maxConcurrency?: number;
|
|
42
|
+
/** P2 dynamic-workflow discriminator. Default "static" (the .workflow.md step-list model).
|
|
43
|
+
* "dynamic" = the workflow is a JS/TS script (.dwf.ts) run via dynamic-workflow-runner.
|
|
44
|
+
* Backward-compatible: absent = "static". */
|
|
45
|
+
runtime?: "static" | "dynamic";
|
|
46
|
+
/** For runtime:"dynamic" — relative/absolute path to the .dwf.ts script. Unused for static. */
|
|
47
|
+
dynamicScript?: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A dynamic workflow (runtime === "dynamic"). steps is empty — the script is the source of truth. */
|
|
51
|
+
export interface DynamicWorkflowConfig extends WorkflowConfig {
|
|
52
|
+
runtime: "dynamic";
|
|
53
|
+
dynamicScript: string;
|
|
54
|
+
steps: [];
|
|
42
55
|
}
|
|
@@ -4,7 +4,7 @@ description: Parallel research team for multi-project/source audits
|
|
|
4
4
|
workspaceMode: single
|
|
5
5
|
defaultWorkflow: parallel-research
|
|
6
6
|
maxConcurrency: 4
|
|
7
|
-
triggers:
|
|
7
|
+
triggers: deep reading, deep read, deep research, source audit, multiple projects, parallel research, pi-*
|
|
8
8
|
category: research
|
|
9
9
|
cost: cheap
|
|
10
10
|
---
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hello.dwf.ts — Minimal reference dynamic-workflow script (P2).
|
|
3
|
+
*
|
|
4
|
+
* Usage: place under `.crew/workflows/hello.dwf.ts`, then
|
|
5
|
+
* team action='run' workflow='hello' goal='Greet the user warmly.'
|
|
6
|
+
*
|
|
7
|
+
* Demonstrates the WorkflowCtx surface: one ctx.agent() call + ctx.setResult().
|
|
8
|
+
* See 07-PLAN.md v3 §3.1 and 00-SPEC.md §3.2.
|
|
9
|
+
*/
|
|
10
|
+
import type { WorkflowCtx } from "../src/runtime/dynamic-workflow-context.ts";
|
|
11
|
+
|
|
12
|
+
export default async function (ctx: WorkflowCtx): Promise<void> {
|
|
13
|
+
const greeting = await ctx.agent({
|
|
14
|
+
role: "executor",
|
|
15
|
+
prompt: `Compose a single-line warm greeting. Context: ${ctx.goal ?? "(no goal)"}`,
|
|
16
|
+
maxTurns: 2,
|
|
17
|
+
});
|
|
18
|
+
// Only ctx.setResult() reaches the main context.
|
|
19
|
+
ctx.setResult("results/greeting.txt", { ok: greeting.ok, model: "executor" });
|
|
20
|
+
// Note: in this trivial example the artifact path is illustrative; production scripts
|
|
21
|
+
// would use ctx.agent()'s returned artifactPath. Here we just surface the agent's text
|
|
22
|
+
// via the summary (runDynamicWorkflow reads the final artifact or falls back).
|
|
23
|
+
void greeting;
|
|
24
|
+
}
|