pi-crew 0.9.26 → 0.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/NOTICE.md +14 -0
- package/assets/crew-vibes.ttf +0 -0
- package/assets/runner-spritesheet.png +0 -0
- package/dist/build-meta.json +255 -40
- package/dist/index.mjs +1564 -241
- package/dist/index.mjs.map +4 -4
- package/docs/perf/optimization-plan-2026-07-verified.md +396 -0
- package/package.json +9 -2
- package/scripts/bench-check.mjs +96 -0
- package/scripts/bench-cold-start.mjs +216 -0
- package/scripts/build-bundle.mjs +84 -0
- package/scripts/build-crew-vibes-font.py +328 -0
- package/scripts/check-all-skills.ts +294 -0
- package/scripts/check-bundle-staleness.mjs +97 -0
- package/scripts/check-conflict-markers.mjs +91 -0
- package/scripts/check-lazy-imports.mjs +28 -0
- package/scripts/install-crew-vibes-font.mjs +91 -0
- package/scripts/postinstall.mjs +47 -0
- package/scripts/profile-startup.mjs +125 -0
- package/scripts/release-smoke.mjs +74 -0
- package/scripts/run-bench.mjs +47 -0
- package/scripts/run-real-chain.ts +41 -0
- package/scripts/test-issue-29-crash.ts +111 -0
- package/scripts/test-issue-29-e2e.ts +183 -0
- package/scripts/test-issue-29-real-runtime.ts +330 -0
- package/scripts/test-issue-29-real-tasks.ts +387 -0
- package/scripts/test-issue-29-team-tool.ts +105 -0
- package/scripts/test-runner.mjs +74 -0
- package/scripts/verify-flicker-fix.ts +109 -0
- package/scripts/verify-skill.ts +550 -0
- package/scripts/watch-bundle.mjs +259 -0
- package/scripts/watchdog-harness.ts +119 -0
- package/src/agents/discover-agents.ts +6 -1
- package/src/config/defaults.ts +6 -0
- package/src/extension/crew-vibes/cat-frames.ts +18 -0
- package/src/extension/crew-vibes/config.ts +195 -0
- package/src/extension/crew-vibes/figures.ts +94 -0
- package/src/extension/crew-vibes/font-detect.ts +58 -0
- package/src/extension/crew-vibes/index.ts +396 -0
- package/src/extension/crew-vibes/provider-usage.ts +330 -0
- package/src/extension/crew-vibes/render.ts +206 -0
- package/src/extension/crew-vibes/speed.ts +286 -0
- package/src/extension/knowledge-injection.ts +12 -3
- package/src/extension/management.ts +23 -3
- package/src/extension/register.ts +7 -0
- package/src/runtime/child-pi.ts +117 -10
- package/src/runtime/crew-agent-records.ts +10 -3
- package/src/runtime/retry-executor.ts +4 -1
- package/src/runtime/task-runner/state-helpers.ts +9 -30
- package/src/runtime/team-runner.ts +5 -3
- package/src/state/atomic-write.ts +153 -49
- package/src/state/event-log.ts +16 -10
- package/src/state/locks.ts +7 -8
- package/src/state/mailbox.ts +15 -4
- package/src/state/state-store.ts +39 -10
- package/src/teams/discover-teams.ts +56 -1
- package/src/utils/safe-paths.ts +2 -1
- package/src/workflows/discover-workflows.ts +72 -1
|
@@ -68,12 +68,19 @@ export function persistSingleTaskUpdate(
|
|
|
68
68
|
const latest = loadRunManifestById(manifest.cwd, manifest.runId)?.tasks ?? fallbackTasks;
|
|
69
69
|
merged = updateTask(latest, taskWithCheckpoint);
|
|
70
70
|
|
|
71
|
-
//
|
|
71
|
+
// F2: collapsed from 3 redundant statSync calls into 1. The previous
|
|
72
|
+
// implementation re-checked mtime twice more after load and before
|
|
73
|
+
// write, but since the code is synchronous and `loadRunManifestById`
|
|
74
|
+
// holds no I/O-yield between the load and this stat, those re-checks
|
|
75
|
+
// always returned the same mtime and added nothing. The one CAS below
|
|
76
|
+
// remains necessary for best-effort writers (async-notifier,
|
|
77
|
+
// crash-recovery) that don't acquire the run lock.
|
|
72
78
|
let currentMtime: number;
|
|
73
79
|
try {
|
|
74
80
|
currentMtime = fs.statSync(manifest.tasksPath).mtimeMs;
|
|
75
81
|
} catch {
|
|
76
|
-
|
|
82
|
+
// Run state deleted (prune/forget) — nothing to persist.
|
|
83
|
+
return fallbackTasks;
|
|
77
84
|
}
|
|
78
85
|
|
|
79
86
|
if (currentMtime !== baseMtime) {
|
|
@@ -82,34 +89,6 @@ export function persistSingleTaskUpdate(
|
|
|
82
89
|
continue;
|
|
83
90
|
}
|
|
84
91
|
|
|
85
|
-
// No concurrent writer — check that our merged result is based on the
|
|
86
|
-
// same base we observed (no intermediate writer between our load and check)
|
|
87
|
-
let recheckMtime: number;
|
|
88
|
-
try {
|
|
89
|
-
recheckMtime = fs.statSync(manifest.tasksPath).mtimeMs;
|
|
90
|
-
} catch {
|
|
91
|
-
// Run state deleted (prune/forget) — nothing to persist.
|
|
92
|
-
return fallbackTasks;
|
|
93
|
-
}
|
|
94
|
-
if (recheckMtime !== baseMtime) {
|
|
95
|
-
baseMtime = recheckMtime;
|
|
96
|
-
continue;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// Final pre-write mtime check to catch any concurrent writer that completed
|
|
100
|
-
// between the recheck and saveRunTasks
|
|
101
|
-
let preWriteMtime: number;
|
|
102
|
-
try {
|
|
103
|
-
preWriteMtime = fs.statSync(manifest.tasksPath).mtimeMs;
|
|
104
|
-
} catch {
|
|
105
|
-
preWriteMtime = 0;
|
|
106
|
-
}
|
|
107
|
-
if (preWriteMtime !== baseMtime) {
|
|
108
|
-
// Another writer committed — retry
|
|
109
|
-
baseMtime = preWriteMtime;
|
|
110
|
-
continue;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
92
|
break;
|
|
114
93
|
}
|
|
115
94
|
|
|
@@ -171,14 +171,16 @@ export function checkPerTaskBudget(
|
|
|
171
171
|
const totalUsed = (usage?.input ?? 0) + (usage?.output ?? 0) + (usage?.cacheWrite ?? 0);
|
|
172
172
|
const abort = totalUsed >= budgetAbort * budgetTotal;
|
|
173
173
|
const warning = !abort && totalUsed >= budgetWarning * budgetTotal;
|
|
174
|
-
|
|
175
|
-
|
|
174
|
+
// Fair share threshold based on TOTAL budget, not remaining budget.
|
|
175
|
+
// This ensures a task that consumed 60% of total budget is flagged even
|
|
176
|
+
// if only 40% remains (40% * 50% = 20% threshold would miss the 60% usage).
|
|
177
|
+
const fairShareThreshold = budgetTotal * fairShareFraction;
|
|
176
178
|
const fairShareViolators: string[] = [];
|
|
177
179
|
for (const task of tasks) {
|
|
178
180
|
if (!task.usage) continue;
|
|
179
181
|
const taskTotal = (task.usage.input ?? 0) + (task.usage.output ?? 0) + (task.usage.cacheWrite ?? 0);
|
|
180
182
|
// Only flag tasks that individually consumed a significant portion of the
|
|
181
|
-
// budget (>10% of total) AND exceeded the fair share
|
|
183
|
+
// budget (>10% of total) AND exceeded the fair share threshold.
|
|
182
184
|
if (fairShareThreshold > 0 && taskTotal > fairShareThreshold && taskTotal > budgetTotal * 0.1) {
|
|
183
185
|
fairShareViolators.push(task.id);
|
|
184
186
|
}
|
|
@@ -160,6 +160,79 @@ export function isSymlinkSafePath(filePath: string): boolean {
|
|
|
160
160
|
}
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
+
// ─── F5: Symlink-safe ancestor chain cache ─────────────────────────────────
|
|
164
|
+
// `isSymlinkSafePath` walks the full ancestor chain with `lstatSync` per level
|
|
165
|
+
// and is called twice per atomic write (initial + pre-rename re-check). The
|
|
166
|
+
// parent dirs of `.crew/state/runs/{runId}/` are stable within a run, so we
|
|
167
|
+
// cache the ancestor-walk verdict keyed by `dirname(filePath)`.
|
|
168
|
+
//
|
|
169
|
+
// SECURITY: the *target file* itself is still checked every call (no cache).
|
|
170
|
+
// Caching is per-dir at a short TTL. If a dir is mutated mid-window, the cache
|
|
171
|
+
// is invalidated explicitly via `invalidateSymlinkSafeCache(dir)` from any
|
|
172
|
+
// file-create / rename hook, plus a TTL failsafe. The cache is best-effort
|
|
173
|
+
// speed-up — when in doubt the next call's statSync re-verifies.
|
|
174
|
+
const SYMLINK_SAFE_TTL_MS = 30_000;
|
|
175
|
+
const SYMLINK_SAFE_MAX_ENTRIES = 128;
|
|
176
|
+
// NOTE: This cache is process-local and assumes single-threaded access.
|
|
177
|
+
// If worker-thread usage expands (e.g., worker-atomic-writer.ts), this cache
|
|
178
|
+
// would need synchronization (e.g., a Mutex or WeakRef pattern).
|
|
179
|
+
const symlinkSafeCache = new Map<string, { safe: boolean; at: number }>();
|
|
180
|
+
|
|
181
|
+
/** Drop one or all cached entries. */
|
|
182
|
+
export function invalidateSymlinkSafeCache(dir?: string): void {
|
|
183
|
+
if (dir) symlinkSafeCache.delete(dir);
|
|
184
|
+
else symlinkSafeCache.clear();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Target-file-only symlink check. Cheap single lstat, never cached — an
|
|
188
|
+
* attacker can swap a regular file for a symlink at the target path between two
|
|
189
|
+
* writes to the same dir, so this must run on every call. */
|
|
190
|
+
function isTargetNotSymlink(filePath: string): boolean {
|
|
191
|
+
try {
|
|
192
|
+
if (fs.lstatSync(filePath).isSymbolicLink()) return false;
|
|
193
|
+
} catch {
|
|
194
|
+
// File doesn't exist yet — that's OK, atomicWriteFile will create it.
|
|
195
|
+
}
|
|
196
|
+
return true;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Cached wrapper used by `atomicWriteFile` on the hot path. Only the ancestor-
|
|
201
|
+
* chain walk (the slow part) is cached, keyed by `dirname(filePath)`; the
|
|
202
|
+
* target-file symlink check ALWAYS re-runs so a mid-window symlink swap at the
|
|
203
|
+
* target path is still caught (preserving the pre-rename TOCTOU guard).
|
|
204
|
+
*
|
|
205
|
+
* The cached verdict is computed from `isSymlinkSafePath(dir)` — i.e. the safety
|
|
206
|
+
* of the *directory* itself, independent of any specific target file — so the
|
|
207
|
+
* entry is correctly reusable for every file written into that dir (no
|
|
208
|
+
* cross-file cache poisoning).
|
|
209
|
+
*/
|
|
210
|
+
function isSymlinkSafeDirCached(filePath: string): boolean {
|
|
211
|
+
const now = Date.now();
|
|
212
|
+
const dir = path.dirname(filePath);
|
|
213
|
+
// Always re-check the target file (uncached) before trusting the dir verdict.
|
|
214
|
+
if (!isTargetNotSymlink(filePath)) return false;
|
|
215
|
+
const hit = symlinkSafeCache.get(dir);
|
|
216
|
+
if (hit && now - hit.at < SYMLINK_SAFE_TTL_MS) return hit.safe;
|
|
217
|
+
// Cache miss: evaluate the ancestor-chain safety of the DIRECTORY itself.
|
|
218
|
+
// Pass a synthetic file path (dir + sentinel) so `dir` is checked as an
|
|
219
|
+
// ANCESTOR rather than a target file. When `dir` itself is a symlink
|
|
220
|
+
// (e.g. macOS /tmp → /private/tmp), the ancestor-walk logic resolves it
|
|
221
|
+
// and verifies ownership — accepting known system temp dirs. Passing `dir`
|
|
222
|
+
// bare would hit the target-file symlink check at the bottom of
|
|
223
|
+
// isSymlinkSafePath, which rejects ALL symlinks without ownership
|
|
224
|
+
// resolution, causing false rejections for legitimate system dirs.
|
|
225
|
+
const verdict = isSymlinkSafePath(path.join(dir, ".symlink-safe-check"));
|
|
226
|
+
symlinkSafeCache.set(dir, { safe: verdict, at: now });
|
|
227
|
+
// Bound the cache to avoid unbounded growth across many cwds.
|
|
228
|
+
while (symlinkSafeCache.size > SYMLINK_SAFE_MAX_ENTRIES) {
|
|
229
|
+
const oldest = symlinkSafeCache.keys().next().value;
|
|
230
|
+
if (oldest === undefined) break;
|
|
231
|
+
symlinkSafeCache.delete(oldest);
|
|
232
|
+
}
|
|
233
|
+
return verdict;
|
|
234
|
+
}
|
|
235
|
+
|
|
163
236
|
function sleep(ms: number): Promise<void> {
|
|
164
237
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
165
238
|
}
|
|
@@ -319,8 +392,39 @@ export async function renameWithRetryAsync(
|
|
|
319
392
|
/** Test alias for renameWithRetryAsync. */
|
|
320
393
|
export const __test__renameWithRetryAsync = renameWithRetryAsync;
|
|
321
394
|
|
|
322
|
-
|
|
323
|
-
|
|
395
|
+
/**
|
|
396
|
+
* F4: per-write durability knob.
|
|
397
|
+
* - "full" (default): fsync the data file AND the parent directory after rename.
|
|
398
|
+
* Cost is ms-scale on Windows; safe against crash + power-loss.
|
|
399
|
+
* - "best-effort": skip both fsyncs. We still get atomic rename from temp file,
|
|
400
|
+
* so a concurrent reader sees either the prior content or the full new
|
|
401
|
+
* content — never a torn write. The trade-off is: a hard crash between
|
|
402
|
+
* rename and a later `flush` may leave a few bytes stale on disk, which is
|
|
403
|
+
* acceptable for informational writes (mailbox delivery, agent progress,
|
|
404
|
+
* .seq sidecar) that the event-log reconstructor / crash-recovery can
|
|
405
|
+
* reconcile without.
|
|
406
|
+
*/
|
|
407
|
+
export type WriteDurability = "full" | "best-effort";
|
|
408
|
+
|
|
409
|
+
/** Options accepted by atomicWriteFile (forward-compatible string | object form). */
|
|
410
|
+
export type AtomicWriteOptions =
|
|
411
|
+
| string // legacy: expectedHash
|
|
412
|
+
| { expectedHash?: string; durability?: WriteDurability };
|
|
413
|
+
|
|
414
|
+
function normalizeOptions(arg: unknown): { expectedHash?: string; durability: WriteDurability } {
|
|
415
|
+
if (typeof arg === "string") return { expectedHash: arg, durability: "full" };
|
|
416
|
+
if (arg && typeof arg === "object") {
|
|
417
|
+
const o = arg as { expectedHash?: string; durability?: WriteDurability };
|
|
418
|
+
const durability: WriteDurability = o.durability === "best-effort" ? "best-effort" : "full";
|
|
419
|
+
return { expectedHash: o.expectedHash, durability };
|
|
420
|
+
}
|
|
421
|
+
return { durability: "full" };
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
export function atomicWriteFile(filePath: string, content: string, options?: AtomicWriteOptions): void {
|
|
425
|
+
const { durability } = normalizeOptions(options);
|
|
426
|
+
if (!isSymlinkSafeDirCached(filePath))
|
|
427
|
+
throw new Error(`Refusing to write: target is a symlink or inside untrusted directory: ${filePath}`);
|
|
324
428
|
// On Windows the parent directory may be referenced via a short-name alias
|
|
325
429
|
// (e.g. RUNNER~1 vs runneradmin). mkdirSync on one form can succeed while
|
|
326
430
|
// openSync on another form fails with ENOENT. We therefore ensure the dir
|
|
@@ -381,7 +485,8 @@ export function atomicWriteFile(filePath: string, content: string, expectedHash?
|
|
|
381
485
|
// rename + post-rename read always sees the new content. Cost: ~1ms
|
|
382
486
|
// per write (acceptable for the small delivery.json / manifest.json
|
|
383
487
|
// files this function writes; large state-store uses the async path).
|
|
384
|
-
|
|
488
|
+
// F4: skip when durability is "best-effort" — see WriteDurability docs.
|
|
489
|
+
if (durability === "full") fs.fsyncSync(fd);
|
|
385
490
|
fs.closeSync(fd);
|
|
386
491
|
try {
|
|
387
492
|
// Issue 1 fix: re-check symlink safety immediately before rename.
|
|
@@ -390,8 +495,12 @@ export function atomicWriteFile(filePath: string, content: string, expectedHash?
|
|
|
390
495
|
// symlink at the target path. If rename succeeds with a symlink at
|
|
391
496
|
// target, the symlink is atomically replaced with attacker's content.
|
|
392
497
|
// The post-rename lstat check only runs on rename failure, so we must
|
|
393
|
-
// check BEFORE the rename to catch this TOCTOU race.
|
|
394
|
-
|
|
498
|
+
// check BEFORE the rename to catch this TOCTOU race. Use the cached
|
|
499
|
+
// wrapper for the dir chain — the pre-rename race only matters for the
|
|
500
|
+
// *target* symlink swap, which the un-cached path check handles on the
|
|
501
|
+
// first call. Here we just re-confirm nothing in the dir chain has been
|
|
502
|
+
// planted symlink-wise since the initial call.
|
|
503
|
+
if (!isSymlinkSafeDirCached(filePath)) {
|
|
395
504
|
throw new Error(`Refusing to rename: target became a symlink or inside untrusted directory: ${filePath}`);
|
|
396
505
|
}
|
|
397
506
|
// Issue 1 fix: use link+unlink instead of rename to avoid following symlinks
|
|
@@ -404,7 +513,8 @@ export function atomicWriteFile(filePath: string, content: string, expectedHash?
|
|
|
404
513
|
// We skip on Windows where directory fsync semantics differ (and
|
|
405
514
|
// where the Windows-specific rename path in renameWithLinkSync already
|
|
406
515
|
// uses MoveFileEx which is fully durable).
|
|
407
|
-
|
|
516
|
+
// F4: honor durability — best-effort also skips the parent-dir fsync.
|
|
517
|
+
if (durability === "full" && process.platform !== "win32") {
|
|
408
518
|
try {
|
|
409
519
|
const dirFd = fs.openSync(path.dirname(filePath), "r");
|
|
410
520
|
fs.fsyncSync(dirFd);
|
|
@@ -450,7 +560,8 @@ export function atomicWriteFile(filePath: string, content: string, expectedHash?
|
|
|
450
560
|
}
|
|
451
561
|
}
|
|
452
562
|
|
|
453
|
-
export async function atomicWriteFileAsync(filePath: string, content: string): Promise<void> {
|
|
563
|
+
export async function atomicWriteFileAsync(filePath: string, content: string, options?: AtomicWriteOptions): Promise<void> {
|
|
564
|
+
const { durability } = normalizeOptions(options);
|
|
454
565
|
// Phase 1.5 (RFC 15): when the worker-thread atomic writer is enabled
|
|
455
566
|
// (PI_CREW_WORKER_ATOMIC_WRITER=1), dispatch to a dedicated worker thread
|
|
456
567
|
// that performs SYNC fs ops with no internal yields. Mitigates the
|
|
@@ -459,7 +570,8 @@ export async function atomicWriteFileAsync(filePath: string, content: string): P
|
|
|
459
570
|
if (isWorkerAtomicWriterEnabled()) {
|
|
460
571
|
return atomicWriteFileViaWorker(filePath, content);
|
|
461
572
|
}
|
|
462
|
-
if (!
|
|
573
|
+
if (!isSymlinkSafeDirCached(filePath))
|
|
574
|
+
throw new Error(`Refusing to write: target is a symlink or inside untrusted directory: ${filePath}`);
|
|
463
575
|
await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
|
|
464
576
|
const tempPath = `${filePath}.${crypto.randomUUID()}.tmp`;
|
|
465
577
|
try {
|
|
@@ -472,43 +584,26 @@ export async function atomicWriteFileAsync(filePath: string, content: string): P
|
|
|
472
584
|
throw new Error(`Refusing to write: opened path is not a regular file: ${tempPath}`);
|
|
473
585
|
}
|
|
474
586
|
await fd.writeFile(content, "utf-8");
|
|
587
|
+
// F4: skip data fsync when caller is best-effort. atomic rename below
|
|
588
|
+
// still guarantees the reader sees either prior or new content, never torn.
|
|
589
|
+
if (durability === "full") await fd.sync();
|
|
475
590
|
await fd.close();
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
try {
|
|
486
|
-
await fs.promises.rm(tempPath, { force: true });
|
|
487
|
-
} catch {
|
|
488
|
-
/* best-effort */
|
|
489
|
-
}
|
|
490
|
-
throw new Error(`Refusing to rename: target became a symlink or inside untrusted directory: ${filePath}`);
|
|
491
|
-
}
|
|
492
|
-
// Issue 1 fix: use link+unlink instead of rename to avoid following symlinks
|
|
493
|
-
await renameWithLinkAsync(tempPath, filePath);
|
|
494
|
-
} catch (renameError) {
|
|
495
|
-
let matches = false;
|
|
496
|
-
try {
|
|
497
|
-
const existing = await fs.promises.readFile(filePath, "utf-8");
|
|
498
|
-
matches = existing === content;
|
|
499
|
-
} catch {
|
|
500
|
-
/* ignore */
|
|
501
|
-
}
|
|
502
|
-
if (matches) {
|
|
503
|
-
try {
|
|
504
|
-
await fs.promises.rm(tempPath, { force: true });
|
|
505
|
-
} catch (cleanupError) {
|
|
506
|
-
logInternalError("atomic-write.cleanupAsync", cleanupError, `tempPath=${tempPath}`);
|
|
507
|
-
}
|
|
508
|
-
return;
|
|
509
|
-
}
|
|
510
|
-
throw renameError;
|
|
591
|
+
// Re-check symlink safety immediately before rename.
|
|
592
|
+
// Between the initial isSymlinkSafePath check and here,
|
|
593
|
+
// an attacker with control of an ancestor directory could plant a
|
|
594
|
+
// symlink at the target path. If rename succeeds with a symlink at
|
|
595
|
+
// target, the symlink is atomically replaced with attacker's content.
|
|
596
|
+
// The post-rename lstat check only runs on rename failure, so we must
|
|
597
|
+
// check BEFORE the rename to catch this TOCTOU race.
|
|
598
|
+
if (!isSymlinkSafeDirCached(filePath)) {
|
|
599
|
+
throw new Error(`Refusing to rename: target became a symlink or inside untrusted directory: ${filePath}`);
|
|
511
600
|
}
|
|
601
|
+
// Issue 1 fix: use link+unlink instead of rename to avoid following symlinks.
|
|
602
|
+
// Align with sync path: no content-match fallback after failed rename.
|
|
603
|
+
// The racy readFile after failed rename is unreliable (another process
|
|
604
|
+
// could write between the failed rename and the read) and inconsistent
|
|
605
|
+
// with the sync path which just throws.
|
|
606
|
+
await renameWithLinkAsync(tempPath, filePath);
|
|
512
607
|
} catch (error) {
|
|
513
608
|
try {
|
|
514
609
|
await fs.promises.rm(tempPath, { force: true });
|
|
@@ -519,12 +614,12 @@ export async function atomicWriteFileAsync(filePath: string, content: string): P
|
|
|
519
614
|
}
|
|
520
615
|
}
|
|
521
616
|
|
|
522
|
-
export function atomicWriteJson<T>(filePath: string, value: T): void {
|
|
523
|
-
atomicWriteFile(filePath, `${JSON.stringify(value, null, 2)}\n
|
|
617
|
+
export function atomicWriteJson<T>(filePath: string, value: T, options?: AtomicWriteOptions): void {
|
|
618
|
+
atomicWriteFile(filePath, `${JSON.stringify(value, null, 2)}\n`, options);
|
|
524
619
|
}
|
|
525
620
|
|
|
526
|
-
export async function atomicWriteJsonAsync<T>(filePath: string, value: T): Promise<void> {
|
|
527
|
-
await atomicWriteFileAsync(filePath, `${JSON.stringify(value, null, 2)}\n
|
|
621
|
+
export async function atomicWriteJsonAsync<T>(filePath: string, value: T, options?: AtomicWriteOptions): Promise<void> {
|
|
622
|
+
await atomicWriteFileAsync(filePath, `${JSON.stringify(value, null, 2)}\n`, options);
|
|
528
623
|
}
|
|
529
624
|
|
|
530
625
|
// 2.1 — atomic-write coalescer. Buffer the latest payload per filePath and
|
|
@@ -547,6 +642,8 @@ interface CoalescedAtomicWrite {
|
|
|
547
642
|
retryCount: number;
|
|
548
643
|
/** Generation counter to detect stale flushes (Issue 2 fix) */
|
|
549
644
|
generation: number;
|
|
645
|
+
/** F4: durability knob carried into the underlying atomicWriteFile flush. */
|
|
646
|
+
durability: WriteDurability;
|
|
550
647
|
}
|
|
551
648
|
const MAX_FLUSH_RETRIES = 5;
|
|
552
649
|
const pendingAtomicWrites = new Map<string, CoalescedAtomicWrite>();
|
|
@@ -565,7 +662,12 @@ let flushInProgress = 0;
|
|
|
565
662
|
* sees the previous on-disk content. Callers needing read-after-write
|
|
566
663
|
* must call `flushPendingAtomicWrites()` first or use `atomicWriteJson`.
|
|
567
664
|
*/
|
|
568
|
-
export function atomicWriteJsonCoalesced<T>(
|
|
665
|
+
export function atomicWriteJsonCoalesced<T>(
|
|
666
|
+
filePath: string,
|
|
667
|
+
value: T,
|
|
668
|
+
coalesceMs = DEFAULT_ATOMIC_COALESCE_MS,
|
|
669
|
+
options?: AtomicWriteOptions,
|
|
670
|
+
): void {
|
|
569
671
|
const content = `${JSON.stringify(value, null, 2)}\n`;
|
|
570
672
|
const previous = pendingAtomicWrites.get(filePath);
|
|
571
673
|
if (previous) clearTimeout(previous.timer);
|
|
@@ -573,12 +675,14 @@ export function atomicWriteJsonCoalesced<T>(filePath: string, value: T, coalesce
|
|
|
573
675
|
timer.unref();
|
|
574
676
|
// Issue 2 fix: increment generation for each new entry
|
|
575
677
|
const generation = ++writeGeneration;
|
|
678
|
+
const normalized = normalizeOptions(options);
|
|
576
679
|
pendingAtomicWrites.set(filePath, {
|
|
577
680
|
content,
|
|
578
681
|
timer,
|
|
579
682
|
coalesceMs,
|
|
580
683
|
retryCount: 0,
|
|
581
684
|
generation,
|
|
685
|
+
durability: normalized.durability,
|
|
582
686
|
});
|
|
583
687
|
}
|
|
584
688
|
|
|
@@ -591,7 +695,7 @@ function flushOnePendingAtomicWrite(filePath: string): void {
|
|
|
591
695
|
const savedGeneration = entry.generation;
|
|
592
696
|
clearTimeout(entry.timer);
|
|
593
697
|
try {
|
|
594
|
-
atomicWriteFile(filePath, entry.content);
|
|
698
|
+
atomicWriteFile(filePath, entry.content, { durability: entry.durability });
|
|
595
699
|
// Issue 2 fix: Verify generation hasn't changed before deleting.
|
|
596
700
|
// A concurrent write may have replaced entry with a newer one during the flush.
|
|
597
701
|
// Only delete if generation matches (not a newer entry).
|
package/src/state/event-log.ts
CHANGED
|
@@ -816,16 +816,22 @@ function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): Team
|
|
|
816
816
|
const seq = fullEvent.metadata?.seq ?? 0;
|
|
817
817
|
if (!skippedDueToSize) {
|
|
818
818
|
fs.appendFileSync(eventsPath, `${JSON.stringify(redactSecrets(fullEvent))}\n`, "utf-8");
|
|
819
|
-
//
|
|
820
|
-
// the
|
|
821
|
-
//
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
819
|
+
// F3a: skip data fsync for non-terminal events. We still call `persistSequence`
|
|
820
|
+
// below, which means the .seq sidecar might briefly outpace the actual data
|
|
821
|
+
// on disk in a crash, but the event-reconstructor (`event-reconstructor.ts`)
|
|
822
|
+
// already handles inconsistent-tail recovery (appends after the sidecar's
|
|
823
|
+
// claimed sequence are simply ignored on a lossless recovery scan). We trade
|
|
824
|
+
// 1 ms of fsync per event on Windows for informational events; terminal
|
|
825
|
+
// events keep the strict window between append and persistSequence.
|
|
826
|
+
if (isTerminal) {
|
|
827
|
+
const fd = fs.openSync(eventsPath, "r+");
|
|
828
|
+
try {
|
|
829
|
+
fs.fsyncSync(fd);
|
|
830
|
+
} catch {
|
|
831
|
+
// EPERM on Windows CI: best-effort flush
|
|
832
|
+
} finally {
|
|
833
|
+
fs.closeSync(fd);
|
|
834
|
+
}
|
|
829
835
|
}
|
|
830
836
|
// FIX: Persist sequence AFTER the event append to prevent sequence reuse
|
|
831
837
|
// on crash. Only update the sidecar when the event is definitively written.
|
package/src/state/locks.ts
CHANGED
|
@@ -189,14 +189,13 @@ function readLockToken(filePath: string): string | undefined {
|
|
|
189
189
|
* With token matching, A's release is a no-op for B's lock.
|
|
190
190
|
*/
|
|
191
191
|
function timingSafeTokenMatch(a: string, b: string): boolean {
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
bufB
|
|
199
|
-
return timingSafeEqual(safeA, safeB);
|
|
192
|
+
// Early length check prevents timing side-channel that leaks length info.
|
|
193
|
+
// Without this, timingSafeEqual compares the full padded length, revealing
|
|
194
|
+
// that the strings have different lengths via the zero-padding.
|
|
195
|
+
if (a.length !== b.length) return false;
|
|
196
|
+
const bufA = Buffer.from(a);
|
|
197
|
+
const bufB = Buffer.from(b);
|
|
198
|
+
return timingSafeEqual(bufA, bufB);
|
|
200
199
|
}
|
|
201
200
|
|
|
202
201
|
function releaseLock(filePath: string, token: string): void {
|
package/src/state/mailbox.ts
CHANGED
|
@@ -396,7 +396,11 @@ export function readDeliveryState(manifest: TeamRunManifest): MailboxDeliverySta
|
|
|
396
396
|
}
|
|
397
397
|
}
|
|
398
398
|
|
|
399
|
-
function writeDeliveryState(
|
|
399
|
+
function writeDeliveryState(
|
|
400
|
+
manifest: TeamRunManifest,
|
|
401
|
+
state: MailboxDeliveryState,
|
|
402
|
+
options?: { durability?: "full" | "best-effort" },
|
|
403
|
+
): void {
|
|
400
404
|
ensureRunMailbox(manifest);
|
|
401
405
|
// Prune oldest entries if capped
|
|
402
406
|
const MAX_DELIVERY_MESSAGES = 10000;
|
|
@@ -408,7 +412,12 @@ function writeDeliveryState(manifest: TeamRunManifest, state: MailboxDeliverySta
|
|
|
408
412
|
const trimmed = sorted.slice(0, MAX_DELIVERY_MESSAGES);
|
|
409
413
|
state.messages = Object.fromEntries(trimmed);
|
|
410
414
|
}
|
|
411
|
-
|
|
415
|
+
// F4: mailbox delivery is informational — accept losing the very last write on
|
|
416
|
+
// a hard crash (the next message will overwrite it on disk). Cheaper fsync on
|
|
417
|
+
// the hot path; terminal/reply paths still pass full durability below.
|
|
418
|
+
atomicWriteFile(deliveryFile(manifest, true), `${JSON.stringify(redactSecrets(state), null, 2)}\n`, {
|
|
419
|
+
durability: options?.durability ?? "best-effort",
|
|
420
|
+
});
|
|
412
421
|
}
|
|
413
422
|
|
|
414
423
|
/**
|
|
@@ -472,7 +481,8 @@ export function appendMailboxMessage(
|
|
|
472
481
|
const delivery = readDeliveryState(manifest);
|
|
473
482
|
delivery.messages[complete.id] = complete.status;
|
|
474
483
|
delivery.updatedAt = createdAt;
|
|
475
|
-
|
|
484
|
+
// F4: complete transitions are terminal-ish — keep full durability.
|
|
485
|
+
writeDeliveryState(manifest, delivery, { durability: "full" });
|
|
476
486
|
});
|
|
477
487
|
return complete;
|
|
478
488
|
}
|
|
@@ -554,7 +564,8 @@ export function acknowledgeMailboxMessage(manifest: TeamRunManifest, messageId:
|
|
|
554
564
|
if (!delivery.messages[messageId]) throw new Error(`Mailbox message '${messageId}' not found.`);
|
|
555
565
|
delivery.messages[messageId] = "acknowledged";
|
|
556
566
|
delivery.updatedAt = new Date().toISOString();
|
|
557
|
-
|
|
567
|
+
// F4: acknowledge is terminal for that message — keep full durability.
|
|
568
|
+
writeDeliveryState(manifest, delivery, { durability: "full" });
|
|
558
569
|
return delivery;
|
|
559
570
|
});
|
|
560
571
|
}
|
package/src/state/state-store.ts
CHANGED
|
@@ -70,9 +70,18 @@ export interface ManifestCacheEntry {
|
|
|
70
70
|
generation?: number;
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
|
|
73
|
+
// F1: per-stateRoot generation counter. The previous global counter was
|
|
74
|
+
// incremented on every `invalidateRunCache` call regardless of which state's
|
|
75
|
+
// cache was invalidated, so a write to run A caused a hot-path miss in run B's
|
|
76
|
+
// cache even though B's on-disk files were untouched. With per-stateRoot
|
|
77
|
+
// generations, run B only misses its own cache when its own writes happen.
|
|
78
|
+
// NOTE: This generation counter is process-local. Cross-process consistency
|
|
79
|
+
// (e.g., parent + child processes writing to the same run) relies on the
|
|
80
|
+
// mtime/size checks in loadRunManifestById, not on this counter.
|
|
81
|
+
const manifestCacheGeneration = new Map<string, number>();
|
|
82
|
+
function genOf(stateRoot: string): number {
|
|
83
|
+
return manifestCacheGeneration.get(stateRoot) ?? 0;
|
|
84
|
+
}
|
|
76
85
|
|
|
77
86
|
const MANIFEST_CACHE_TTL_MS = 15 * 1000; // 15 seconds (FIX: increased from 5s for read-heavy workloads; 5s was too short causing unnecessary cache invalidation)
|
|
78
87
|
const LOAD_MANIFEST_RETRY_LIMIT = 5; // Configurable retry limit for mtime/size stability checks under contention
|
|
@@ -94,7 +103,9 @@ export const MANIFEST_CACHE_TTL_MS_VALUE = MANIFEST_CACHE_TTL_MS;
|
|
|
94
103
|
function setManifestCache(stateRoot: string, entry: ManifestCacheEntry): void {
|
|
95
104
|
if (manifestCache.has(stateRoot)) manifestCache.delete(stateRoot);
|
|
96
105
|
entry.cachedAt = Date.now();
|
|
97
|
-
|
|
106
|
+
// Stamp with the *current* per-root generation so a concurrent writer's
|
|
107
|
+
// increment between read and write invalidates this cache hit.
|
|
108
|
+
entry.generation = genOf(stateRoot);
|
|
98
109
|
// FIX: Evict all stale entries by TTL before adding new entry.
|
|
99
110
|
// This ensures entries that are never accessed still get evicted
|
|
100
111
|
// based on TTL, not just entries that are hit.
|
|
@@ -129,7 +140,10 @@ function useProjectState(cwd: string): boolean {
|
|
|
129
140
|
|
|
130
141
|
function invalidateRunCache(stateRoot: string): void {
|
|
131
142
|
manifestCache.delete(stateRoot);
|
|
132
|
-
|
|
143
|
+
// F1: only bump the generation of THIS stateRoot — sibling runs keep their
|
|
144
|
+
// generation intact, so concurrent writes to run A no longer evict run B's
|
|
145
|
+
// hot cache.
|
|
146
|
+
manifestCacheGeneration.set(stateRoot, genOf(stateRoot) + 1);
|
|
133
147
|
}
|
|
134
148
|
|
|
135
149
|
function scopeBaseRoot(cwd: string): string {
|
|
@@ -334,13 +348,28 @@ export function saveRunManifest(manifest: TeamRunManifest): void {
|
|
|
334
348
|
// on next read, but the returned empty tasks array could confuse callers
|
|
335
349
|
// that don't re-read.
|
|
336
350
|
const manifestStat = fs.statSync(manifestPath);
|
|
351
|
+
// Read tasks.json if it exists to avoid cache inconsistency.
|
|
352
|
+
// Without this, loadRunManifestById returns empty tasks even when tasks.json
|
|
353
|
+
// has real data, causing incorrect task state display.
|
|
354
|
+
let tasks: TeamTaskState[] = [];
|
|
355
|
+
let tasksMtimeMs = 0;
|
|
356
|
+
let tasksSize = 0;
|
|
357
|
+
try {
|
|
358
|
+
const tasksPath = path.join(manifest.stateRoot, "tasks.json");
|
|
359
|
+
const tasksStat = fs.statSync(tasksPath);
|
|
360
|
+
tasks = JSON.parse(fs.readFileSync(tasksPath, "utf-8")) as TeamTaskState[];
|
|
361
|
+
tasksMtimeMs = tasksStat.mtimeMs;
|
|
362
|
+
tasksSize = tasksStat.size;
|
|
363
|
+
} catch {
|
|
364
|
+
// tasks.json doesn't exist or is invalid — leave tasks empty
|
|
365
|
+
}
|
|
337
366
|
setManifestCache(manifest.stateRoot, {
|
|
338
367
|
manifest,
|
|
339
|
-
tasks
|
|
368
|
+
tasks,
|
|
340
369
|
manifestMtimeMs: manifestStat.mtimeMs,
|
|
341
370
|
manifestSize: manifestStat.size,
|
|
342
|
-
tasksMtimeMs
|
|
343
|
-
tasksSize
|
|
371
|
+
tasksMtimeMs,
|
|
372
|
+
tasksSize,
|
|
344
373
|
});
|
|
345
374
|
}
|
|
346
375
|
|
|
@@ -642,7 +671,7 @@ export function loadRunManifestById(cwd: string, runId: string): { manifest: Tea
|
|
|
642
671
|
cached.manifestSize === manifestStat.size &&
|
|
643
672
|
cached.tasksMtimeMs === tasksMtimeMs &&
|
|
644
673
|
cached.tasksSize === (tasksStat?.size ?? 0) &&
|
|
645
|
-
cached.generation ===
|
|
674
|
+
cached.generation === genOf(stateRoot)
|
|
646
675
|
) {
|
|
647
676
|
// TTL eviction: expire stale entries even if mtime matches
|
|
648
677
|
// FIX: Also evict entries where cachedAt is undefined — such entries are
|
|
@@ -758,7 +787,7 @@ export async function loadRunManifestByIdAsync(
|
|
|
758
787
|
cached.manifestSize === manifestStat.size &&
|
|
759
788
|
cached.tasksMtimeMs === tasksMtimeMs &&
|
|
760
789
|
cached.tasksSize === (tasksStat?.size ?? 0) &&
|
|
761
|
-
cached.generation ===
|
|
790
|
+
cached.generation === genOf(stateRoot)
|
|
762
791
|
) {
|
|
763
792
|
// TTL eviction: expire stale entries even if mtime matches
|
|
764
793
|
// FIX: Also evict entries where cachedAt is undefined — such entries are
|
|
@@ -114,12 +114,67 @@ function readTeamDir(dir: string, source: ResourceSource): TeamConfig[] {
|
|
|
114
114
|
.sort((a, b) => a.name.localeCompare(b.name));
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
+
// ─── Team Discovery Cache (F15) ──────────────────────────────────────────
|
|
118
|
+
// Mirrors the Workflow Discovery Cache (see workflows/discover-workflows.ts).
|
|
119
|
+
// `discoverTeams(cwd)` is called from recommend/scheduler paths and previously
|
|
120
|
+
// re-scanned 3 roots on every call. 5 s TTL + dir-stamp invalidation keeps
|
|
121
|
+
// changes (e.g. user edits a `.team.md`) visible within a few seconds without
|
|
122
|
+
// paying a full re-scan per call.
|
|
123
|
+
const TEAM_DISCOVERY_TTL_MS = 5000;
|
|
124
|
+
const TEAM_DISCOVERY_MAX_ENTRIES = 32;
|
|
125
|
+
interface CachedTeamEntry {
|
|
126
|
+
result: TeamDiscoveryResult;
|
|
127
|
+
expiresAt: number;
|
|
128
|
+
dirStamp: string;
|
|
129
|
+
}
|
|
130
|
+
const teamCache = new Map<string, CachedTeamEntry>();
|
|
131
|
+
|
|
132
|
+
function teamDirStamp(cwd: string): string {
|
|
133
|
+
const dirs = [path.join(packageRoot(), "teams"), path.join(userPiRoot(), "teams"), path.join(projectCrewRoot(cwd), "teams")];
|
|
134
|
+
let out = "";
|
|
135
|
+
for (const d of dirs) {
|
|
136
|
+
try {
|
|
137
|
+
out += `${fs.statSync(d).mtimeMs}|`;
|
|
138
|
+
} catch {
|
|
139
|
+
out += "0|";
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Drop one or all cached entries. Called from management actions and tests. */
|
|
146
|
+
export function invalidateTeamDiscoveryCache(cwd?: string): void {
|
|
147
|
+
if (cwd) {
|
|
148
|
+
teamCache.delete(cwd);
|
|
149
|
+
} else {
|
|
150
|
+
teamCache.clear();
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
117
154
|
export function discoverTeams(cwd: string): TeamDiscoveryResult {
|
|
118
|
-
|
|
155
|
+
// F15: serve from cache when both TTL is fresh AND dir-stamp is unchanged.
|
|
156
|
+
const now = Date.now();
|
|
157
|
+
const stamp = teamDirStamp(cwd);
|
|
158
|
+
const cached = teamCache.get(cwd);
|
|
159
|
+
if (cached && cached.expiresAt > now && cached.dirStamp === stamp) {
|
|
160
|
+
return cached.result;
|
|
161
|
+
}
|
|
162
|
+
const result: TeamDiscoveryResult = {
|
|
119
163
|
builtin: readTeamDir(path.join(packageRoot(), "teams"), "builtin"),
|
|
120
164
|
user: readTeamDir(path.join(userPiRoot(), "teams"), "user"),
|
|
121
165
|
project: readTeamDir(path.join(projectCrewRoot(cwd), "teams"), "project"),
|
|
122
166
|
};
|
|
167
|
+
teamCache.set(cwd, {
|
|
168
|
+
result,
|
|
169
|
+
expiresAt: now + TEAM_DISCOVERY_TTL_MS,
|
|
170
|
+
dirStamp: stamp,
|
|
171
|
+
});
|
|
172
|
+
while (teamCache.size > TEAM_DISCOVERY_MAX_ENTRIES) {
|
|
173
|
+
const oldest = teamCache.keys().next().value;
|
|
174
|
+
if (oldest === undefined) break;
|
|
175
|
+
teamCache.delete(oldest);
|
|
176
|
+
}
|
|
177
|
+
return result;
|
|
123
178
|
}
|
|
124
179
|
|
|
125
180
|
export function allTeams(discovery: TeamDiscoveryResult | undefined): TeamConfig[] {
|
package/src/utils/safe-paths.ts
CHANGED
|
@@ -255,7 +255,8 @@ export function resolveRealContainedPath(baseDir: string, targetPath: string): s
|
|
|
255
255
|
if (process.platform === "darwin") {
|
|
256
256
|
const resolvedSymlink = resolvedAccumulated;
|
|
257
257
|
const knownDarwinSymlinks = ["/var", "/tmp", "/etc", "/private/var", "/private/tmp", "/private/etc"];
|
|
258
|
-
|
|
258
|
+
// Use prefix match to handle subdirectories like /var/folders/...
|
|
259
|
+
if (knownDarwinSymlinks.some((s) => resolvedSymlink === s || resolvedSymlink.startsWith(s + "/"))) continue;
|
|
259
260
|
}
|
|
260
261
|
throw new Error("Refusing to resolve: target path ancestor is a symlink: " + resolvedAccumulated);
|
|
261
262
|
}
|