pi-crew 0.9.55 → 0.9.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/index.mjs +1153 -857
- package/package.json +1 -2
- package/scripts/smoke-team-perf.ts +43 -0
- package/src/extension/registration/team-tool.ts +2 -1
- package/src/extension/team-tool/param-error.ts +114 -0
- package/src/extension/team-tool/status.ts +6 -1
- package/src/runtime/attention-events.ts +3 -2
- package/src/runtime/child-pi-streams.ts +12 -0
- package/src/runtime/child-pi.ts +20 -7
- package/src/runtime/compact-stages/bounded-tail.ts +84 -0
- package/src/runtime/concurrency.ts +8 -2
- package/src/runtime/crew-agent-records.ts +48 -2
- package/src/runtime/diagnostic-export.ts +3 -2
- package/src/runtime/manifest-cache.ts +5 -1
- package/src/runtime/semaphore.ts +4 -1
- package/src/runtime/supervisor-contact.ts +27 -12
- package/src/runtime/task-runner/child-executor.ts +5 -9
- package/src/runtime/team-runner.ts +62 -37
- package/src/schema/team-tool-schema.ts +19 -11
- package/src/state/event-log-rotation.ts +28 -0
- package/src/state/event-log.ts +39 -13
- package/src/state/state-store.ts +18 -4
- package/src/utils/redaction.ts +39 -0
- package/src/worktree/worktree-manager.ts +23 -8
- package/dist/build-meta.json +0 -16807
|
@@ -57,7 +57,7 @@ import { type ProgressEventSummary, shouldAppendProgressEventUpdate } from "../p
|
|
|
57
57
|
import { DEFAULT_RETRY_POLICY } from "../retry-executor.ts";
|
|
58
58
|
import { runWorker } from "../run-worker.ts";
|
|
59
59
|
import { parseSessionUsage } from "../session-usage.ts";
|
|
60
|
-
import {
|
|
60
|
+
import { recordSupervisorContact, supervisorContactFromEvent } from "../supervisor-contact.ts";
|
|
61
61
|
import { createWorkerHeartbeat, touchWorkerHeartbeat } from "../worker-heartbeat.ts";
|
|
62
62
|
import { createStartupEvidence } from "../worker-startup.ts";
|
|
63
63
|
import type { TaskExecutionResult } from "./post-execution.ts";
|
|
@@ -438,19 +438,15 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
438
438
|
onStdoutLine: (line) => {
|
|
439
439
|
appendCrewAgentOutput(manifest, task.id, line);
|
|
440
440
|
persistHeartbeat();
|
|
441
|
-
// Check for supervisor contact requests from child Pi
|
|
442
|
-
const contact = parseSupervisorContactFromLine(line);
|
|
443
|
-
if (contact) {
|
|
444
|
-
recordSupervisorContact(manifest, {
|
|
445
|
-
runId: manifest.runId,
|
|
446
|
-
...contact,
|
|
447
|
-
});
|
|
448
|
-
}
|
|
449
441
|
},
|
|
450
442
|
onJsonEvent: (event) => {
|
|
451
443
|
// Top-level error boundary: prevent any single event from crashing the task.
|
|
452
444
|
// Errors are logged but processing continues so subsequent events still update state.
|
|
453
445
|
try {
|
|
446
|
+
// P2-25: handle supervisor_contact here (compact pipeline now passes
|
|
447
|
+
// the full payload through); was dead on the old displayLine path.
|
|
448
|
+
const contact = supervisorContactFromEvent(event);
|
|
449
|
+
if (contact) recordSupervisorContact(manifest, { runId: manifest.runId, ...contact });
|
|
454
450
|
appendCrewAgentEvent(manifest, task.id, event);
|
|
455
451
|
if (collectedJsonEvents && event && typeof event === "object" && !Array.isArray(event))
|
|
456
452
|
collectedJsonEvents.push(event as Record<string, unknown>);
|
|
@@ -23,7 +23,7 @@ import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
|
|
|
23
23
|
import { buildSyntheticTerminalEvidence, CrewCancellationError, cancellationReasonFromSignal } from "./cancellation.ts";
|
|
24
24
|
import { buildDispatchUnits, type DispatchUnit, planCoalescedGroups } from "./coalesce-tasks.ts";
|
|
25
25
|
import { type BatchConcurrencyDecision, resolveBatchConcurrency } from "./concurrency.ts";
|
|
26
|
-
import { readCrewAgents, saveCrewAgents } from "./crew-agent-records.ts";
|
|
26
|
+
import { readCrewAgents, saveCrewAgents, saveCrewAgentsCoalesced } from "./crew-agent-records.ts";
|
|
27
27
|
import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
|
|
28
28
|
import { crewHooks } from "./crew-hooks.ts";
|
|
29
29
|
import { appendDeadletter } from "./deadletter.ts";
|
|
@@ -918,8 +918,9 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
918
918
|
await saveRunTasksAsync(manifest, tasks);
|
|
919
919
|
const existingRuntimeByTask = new Map(readCrewAgents(manifest).map((agent) => [agent.taskId, agent.runtime]));
|
|
920
920
|
const globalRuntime = input.runtime?.kind ?? "child-process";
|
|
921
|
+
const taskById = new Map(tasks.map((item) => [item.id, item] as const));
|
|
921
922
|
const runtimeForAgent = (agent: ReturnType<typeof recordsForMaterializedTasks>[number]): CrewRuntimeKind => {
|
|
922
|
-
const task =
|
|
923
|
+
const task = taskById.get(agent.taskId);
|
|
923
924
|
return (
|
|
924
925
|
existingRuntimeByTask.get(agent.taskId) ??
|
|
925
926
|
resolveTaskRuntimeKind(globalRuntime, task?.role ?? agent.role, input.runtimeConfig?.isolationPolicy)
|
|
@@ -1059,14 +1060,17 @@ async function cancelRunFromSignal(ctx: SchedulerContext): Promise<SchedulerDeci
|
|
|
1059
1060
|
return base;
|
|
1060
1061
|
});
|
|
1061
1062
|
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1063
|
+
await Promise.all(
|
|
1064
|
+
cancelledTaskIds.map((taskId) =>
|
|
1065
|
+
appendEventAsync(ctx.manifest.eventsPath, {
|
|
1066
|
+
type: "task.cancelled",
|
|
1067
|
+
runId: ctx.manifest.runId,
|
|
1068
|
+
taskId,
|
|
1069
|
+
message,
|
|
1070
|
+
data: { reason: cancelReason.code },
|
|
1071
|
+
}),
|
|
1072
|
+
),
|
|
1073
|
+
);
|
|
1070
1074
|
ctx.manifest = updateRunStatus(ctx.manifest, "cancelled", message, {
|
|
1071
1075
|
data: { reason: cancelReason.code, cancelledTaskIds },
|
|
1072
1076
|
});
|
|
@@ -1213,9 +1217,9 @@ async function selectDispatchBatch(ctx: SchedulerContext): Promise<SchedulerDeci
|
|
|
1213
1217
|
}
|
|
1214
1218
|
}
|
|
1215
1219
|
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1220
|
+
// W5-4: by-id map once (was O(ready × tasks) via find-per-element).
|
|
1221
|
+
const taskByIdReady = new Map(ctx.tasks.map((t) => [t.id, t] as const));
|
|
1222
|
+
const readyRoles = readyBeforeFilter.map((taskId) => taskByIdReady.get(taskId)?.role).filter((role): role is string => Boolean(role));
|
|
1219
1223
|
const concurrency = resolveBatchConcurrency({
|
|
1220
1224
|
workflowName: ctx.workflow.name,
|
|
1221
1225
|
workflowMaxConcurrency: ctx.workflow.maxConcurrency,
|
|
@@ -1287,9 +1291,8 @@ async function selectDispatchBatch(ctx: SchedulerContext): Promise<SchedulerDeci
|
|
|
1287
1291
|
const approvalPending = isPlanApprovalPending(ctx.manifest);
|
|
1288
1292
|
const dispatchableReady = serializedReady.filter((id) => !inFlightTaskIds.has(id));
|
|
1289
1293
|
const readyIds = approvalPending ? dispatchableReady : dispatchableReady.slice(0, slotsAvailable);
|
|
1290
|
-
const
|
|
1291
|
-
|
|
1292
|
-
.filter((task): task is TeamTaskState => Boolean(task));
|
|
1294
|
+
const taskByIdDispatch = new Map(ctx.tasks.map((t) => [t.id, t] as const));
|
|
1295
|
+
const candidateBatch = readyIds.map((id) => taskByIdDispatch.get(id)).filter((task): task is TeamTaskState => Boolean(task));
|
|
1293
1296
|
const readyBatch = approvalPending ? candidateBatch.filter((task) => !isMutatingTask(task)).slice(0, slotsAvailable) : candidateBatch;
|
|
1294
1297
|
if (readyBatch.length === 0) {
|
|
1295
1298
|
if (ctx.pendingUnits.size > 0) {
|
|
@@ -1360,13 +1363,18 @@ async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBatchDecis
|
|
|
1360
1363
|
concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason,
|
|
1361
1364
|
},
|
|
1362
1365
|
});
|
|
1363
|
-
// Execute before_task_start hooks for the batch
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1366
|
+
// Execute before_task_start hooks for the batch — P1-10: run hooks in
|
|
1367
|
+
// parallel (each may be a subprocess), then apply skip mutations in order.
|
|
1368
|
+
const beforeTaskStartReports = await Promise.all(
|
|
1369
|
+
readyBatch.map((task) =>
|
|
1370
|
+
executeHook("before_task_start", {
|
|
1371
|
+
runId: ctx.manifest.runId,
|
|
1372
|
+
taskId: task.id,
|
|
1373
|
+
cwd: ctx.manifest.cwd,
|
|
1374
|
+
}).then((taskReport) => ({ task, taskReport })),
|
|
1375
|
+
),
|
|
1376
|
+
);
|
|
1377
|
+
for (const { task, taskReport } of beforeTaskStartReports) {
|
|
1370
1378
|
appendHookEvent(ctx.manifest, taskReport);
|
|
1371
1379
|
if (taskReport.outcome === "block") {
|
|
1372
1380
|
ctx.tasks = ctx.tasks.map((t) =>
|
|
@@ -1381,7 +1389,12 @@ async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBatchDecis
|
|
|
1381
1389
|
ctx.manifest = updateRunStatus(ctx.manifest, ctx.manifest.status, `Task '${task.id}' blocked by hook.`);
|
|
1382
1390
|
}
|
|
1383
1391
|
}
|
|
1384
|
-
|
|
1392
|
+
// W5-4: by-id map (was O(readyBatch × tasks) via find-per-element).
|
|
1393
|
+
const ctxTaskById = new Map(ctx.tasks.map((t) => [t.id, t] as const));
|
|
1394
|
+
const batchTasks = readyBatch.filter((task) => {
|
|
1395
|
+
const t = ctxTaskById.get(task.id);
|
|
1396
|
+
return t !== undefined && t.status !== "skipped";
|
|
1397
|
+
});
|
|
1385
1398
|
if (batchTasks.length > 1) {
|
|
1386
1399
|
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1387
1400
|
type: "task.parallel_start",
|
|
@@ -1791,12 +1804,14 @@ async function advanceWorkflowPhases(ctx: SchedulerContext): Promise<void> {
|
|
|
1791
1804
|
existing.push(task.id);
|
|
1792
1805
|
phaseTaskMap.set(task.stepId, existing);
|
|
1793
1806
|
}
|
|
1807
|
+
// W5-4: by-id map once for the phase loop (was O(phases × phaseTasks × tasks)).
|
|
1808
|
+
const taskById = new Map(tasks.map((t) => [t.id, t] as const));
|
|
1794
1809
|
for (let pi = wfMachine.currentPhaseIndex; pi < wfMachine.phases.length; pi++) {
|
|
1795
1810
|
const phase = wfMachine.phases[pi]!;
|
|
1796
1811
|
const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
|
|
1797
1812
|
if (phaseTaskIds.length === 0) continue;
|
|
1798
1813
|
const allTerminal = phaseTaskIds.every((taskId) => {
|
|
1799
|
-
const task =
|
|
1814
|
+
const task = taskById.get(taskId);
|
|
1800
1815
|
return task ? terminalStatuses.has(task.status) : false;
|
|
1801
1816
|
});
|
|
1802
1817
|
if (!allTerminal) break;
|
|
@@ -1816,7 +1831,7 @@ async function advanceWorkflowPhases(ctx: SchedulerContext): Promise<void> {
|
|
|
1816
1831
|
};
|
|
1817
1832
|
// Determine phase transition status based on individual task outcomes
|
|
1818
1833
|
const phaseTasks = phaseTaskIds
|
|
1819
|
-
.map((taskId) =>
|
|
1834
|
+
.map((taskId) => taskById.get(taskId))
|
|
1820
1835
|
.filter((t): t is NonNullable<typeof t> => t !== undefined);
|
|
1821
1836
|
const hasFailedOrCancelled = phaseTasks.some((t) => t.status === "failed" || t.status === "cancelled");
|
|
1822
1837
|
const phaseStatus = hasFailedOrCancelled ? "failed" : "completed";
|
|
@@ -1913,23 +1928,30 @@ async function enforceRunBudget(ctx: SchedulerContext): Promise<SchedulerDecisio
|
|
|
1913
1928
|
|
|
1914
1929
|
// Fair-share warning: flag tasks that consumed >50% of remaining budget
|
|
1915
1930
|
// without killing them mid-execution.
|
|
1931
|
+
const fairShareAppends: Promise<void>[] = [];
|
|
1916
1932
|
for (const violatorId of budgetCheck.fairShareViolators) {
|
|
1917
1933
|
const violator = tasks.find((t) => t.id === violatorId);
|
|
1918
1934
|
if (!violator) continue;
|
|
1919
1935
|
const taskTotal = (violator.usage?.input ?? 0) + (violator.usage?.output ?? 0) + (violator.usage?.cacheWrite ?? 0);
|
|
1920
1936
|
const message = `Task '${violatorId}' consumed ${formatTokens(taskTotal)} (${Math.round((taskTotal / input.budgetTotal) * 100)}% of total budget) — exceeds fair share`;
|
|
1921
1937
|
console.warn(`[team-runner.fair-share] ${message}`);
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1938
|
+
fairShareAppends.push(
|
|
1939
|
+
appendEventAsync(manifest.eventsPath, {
|
|
1940
|
+
type: "task.budget_fair_share",
|
|
1941
|
+
runId: manifest.runId,
|
|
1942
|
+
taskId: violatorId,
|
|
1943
|
+
message,
|
|
1944
|
+
data: {
|
|
1945
|
+
budgetTotal: input.budgetTotal,
|
|
1946
|
+
taskUsage: taskTotal,
|
|
1947
|
+
},
|
|
1948
|
+
}).then(
|
|
1949
|
+
() => {},
|
|
1950
|
+
(error) => logInternalError("team-runner.fair-share-event", error, `taskId=${violatorId}`),
|
|
1951
|
+
),
|
|
1952
|
+
);
|
|
1932
1953
|
}
|
|
1954
|
+
await Promise.all(fairShareAppends);
|
|
1933
1955
|
}
|
|
1934
1956
|
return null;
|
|
1935
1957
|
}
|
|
@@ -2355,7 +2377,10 @@ async function executeTeamRunCore(
|
|
|
2355
2377
|
return { manifest, tasks };
|
|
2356
2378
|
}
|
|
2357
2379
|
await saveRunTasksAsync(manifest, tasks);
|
|
2358
|
-
|
|
2380
|
+
// P0-3: per-batch progress write — coalesced (best-effort, no lock) so the
|
|
2381
|
+
// sync spin-lock (Atomics.wait) doesn't block the event loop every batch.
|
|
2382
|
+
// The terminal saveCrewAgents at closeout remains durable + flushes this.
|
|
2383
|
+
saveCrewAgentsCoalesced(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
2359
2384
|
const completedBatch = tasks.filter((t) => settledTaskIds.includes(t.id));
|
|
2360
2385
|
const batchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
2361
2386
|
kind: "summary",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Type } from "@sinclair/typebox";
|
|
1
|
+
import { type TSchema, Type } from "@sinclair/typebox";
|
|
2
2
|
|
|
3
3
|
// ───────────────────────────────────────────────────────────────────────────
|
|
4
4
|
// API-5 facade split: the 54-action mega-tool schema is split into 5 domain
|
|
@@ -346,17 +346,25 @@ export const ManageDomainParams = Type.Object({ action: manageActions, ...shared
|
|
|
346
346
|
export const AutomateDomainParams = Type.Object({ action: automateActions, ...sharedFields }, { additionalProperties: true });
|
|
347
347
|
|
|
348
348
|
/**
|
|
349
|
-
*
|
|
350
|
-
*
|
|
351
|
-
* (
|
|
349
|
+
* LLM-facing team-tool schema. FLAT single Object (all actions + all shared
|
|
350
|
+
* fields optional) — NOT the former 5-variant Type.Union. The union emitted a
|
|
351
|
+
* giant `anyOf` (each variant repeated all ~30 shared fields) that LLM
|
|
352
|
+
* tool-callers could not reliably satisfy → empty/malformed calls → params
|
|
353
|
+
* dropped → the tool defaulted to `list`. The handler validates per-action
|
|
354
|
+
* fields at runtime (defense-in-depth, team-tool.ts execute()), so the schema
|
|
355
|
+
* only needs to enumerate valid actions + field types. Domain objects above
|
|
356
|
+
* (RunDomainParams etc.) are kept for the facade dispatch + backward compat.
|
|
352
357
|
*/
|
|
353
|
-
export const
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
358
|
+
export const allActionLiterals = ([runActions, statusActions, controlActions, manageActions, automateActions] as TSchema[]).flatMap(
|
|
359
|
+
(set) => set.anyOf ?? [],
|
|
360
|
+
);
|
|
361
|
+
export const TeamToolParams = Type.Object(
|
|
362
|
+
{
|
|
363
|
+
action: Type.Optional(Type.Union(allActionLiterals, { description: ACTION_DESCRIPTION })),
|
|
364
|
+
...sharedFields,
|
|
365
|
+
},
|
|
366
|
+
{ additionalProperties: true },
|
|
367
|
+
);
|
|
360
368
|
|
|
361
369
|
/** Domain discriminator for the facade dispatch. */
|
|
362
370
|
export type TeamDomain = "run" | "status" | "control" | "manage" | "automate";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
2
3
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
3
4
|
import { atomicWriteFile } from "./atomic-write.ts";
|
|
4
5
|
import { readEvents, type TeamEvent, withEventLogLockSync } from "./event-log.ts";
|
|
@@ -250,6 +251,30 @@ export function rotateEventLog(eventsPath: string): boolean {
|
|
|
250
251
|
return withEventLogLockSync(eventsPath, () => rotateEventLogUnlocked(eventsPath));
|
|
251
252
|
}
|
|
252
253
|
|
|
254
|
+
/** H2 retention: delete `<eventsPath>.*.archive.jsonl` files older than the window so
|
|
255
|
+
* they don't accumulate forever (mirrors notification/metric-sink rotateOldFiles). */
|
|
256
|
+
const ARCHIVE_RETENTION_DAYS = 7;
|
|
257
|
+
function sweepOldArchives(eventsPath: string, now = Date.now()): void {
|
|
258
|
+
const dir = path.dirname(eventsPath);
|
|
259
|
+
const base = path.basename(eventsPath);
|
|
260
|
+
let entries: string[];
|
|
261
|
+
try {
|
|
262
|
+
entries = fs.readdirSync(dir);
|
|
263
|
+
} catch {
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
const cutoff = now - ARCHIVE_RETENTION_DAYS * 24 * 60 * 60 * 1000;
|
|
267
|
+
for (const name of entries) {
|
|
268
|
+
if (!name.startsWith(`${base}.`) || !name.endsWith(".archive.jsonl")) continue;
|
|
269
|
+
const archivePath = path.join(dir, name);
|
|
270
|
+
try {
|
|
271
|
+
if (fs.statSync(archivePath).mtimeMs < cutoff) fs.unlinkSync(archivePath);
|
|
272
|
+
} catch (error) {
|
|
273
|
+
logInternalError("event-log.archive-sweep", error, `archivePath=${archivePath}`, "debug");
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
253
278
|
/** Round 24 (BUG 1): the lock-free core of rotation. Assumes the caller
|
|
254
279
|
* already holds the event-log lock (or accepts the unlocked race). */
|
|
255
280
|
export function rotateEventLogUnlocked(eventsPath: string): boolean {
|
|
@@ -278,6 +303,9 @@ export function rotateEventLogUnlocked(eventsPath: string): boolean {
|
|
|
278
303
|
// detect the truncation and reset (re-read from offset 0 of the new
|
|
279
304
|
// file), instead of reading past EOF and missing post-rotation events.
|
|
280
305
|
bumpGenerationUnlocked(eventsPath);
|
|
306
|
+
// H2: retention — sweep archive files older than the window so they don't
|
|
307
|
+
// accumulate forever (mirrors notification/metric-sink rotateOldFiles).
|
|
308
|
+
sweepOldArchives(eventsPath);
|
|
281
309
|
return true;
|
|
282
310
|
} catch (error) {
|
|
283
311
|
logInternalError("event-log.rotate", error, `eventsPath=${eventsPath}`);
|
package/src/state/event-log.ts
CHANGED
|
@@ -69,7 +69,25 @@ const MAX_EVENTS_BYTES = 50 * 1024 * 1024;
|
|
|
69
69
|
|
|
70
70
|
const sequenceCache = new Map<string, { size: number; mtimeMs: number; seq: number; lastAccessMs: number }>();
|
|
71
71
|
const MAX_SEQUENCE_CACHE_ENTRIES = 256;
|
|
72
|
-
|
|
72
|
+
// P0-2: per-eventsPath append counter. Previously a single module-global shared
|
|
73
|
+
// across all runs AND never incremented on the async path, so the async rotation
|
|
74
|
+
// gate (`0 % 100 === 0`) was always true → needsRotation ran on EVERY async append
|
|
75
|
+
// (a full read+parse+rewrite once the log crossed 4 MB). FIFO-bounded to avoid
|
|
76
|
+
// unbounded growth with run count.
|
|
77
|
+
const appendCounters = new Map<string, number>();
|
|
78
|
+
const APPEND_COUNTER_MAX_ENTRIES = 256;
|
|
79
|
+
export function tickAppendCounter(eventsPath: string, inc = 1): boolean {
|
|
80
|
+
const prev = appendCounters.get(eventsPath) ?? 0;
|
|
81
|
+
const next = prev + inc;
|
|
82
|
+
appendCounters.set(eventsPath, next);
|
|
83
|
+
if (appendCounters.size > APPEND_COUNTER_MAX_ENTRIES) {
|
|
84
|
+
const oldest = appendCounters.keys().next().value;
|
|
85
|
+
if (oldest !== undefined) appendCounters.delete(oldest);
|
|
86
|
+
}
|
|
87
|
+
// True when a 100-boundary is crossed (matches the old `next % 100 === 0`
|
|
88
|
+
// semantics for inc=1, and generalizes to batch increments).
|
|
89
|
+
return Math.floor(next / 100) > Math.floor(prev / 100);
|
|
90
|
+
}
|
|
73
91
|
let overflowCounter = 0;
|
|
74
92
|
|
|
75
93
|
/** Simple cross-process lock for an eventsPath to prevent JSONL interleave on concurrent append.
|
|
@@ -318,7 +336,9 @@ function nextSequence(eventsPath: string): number {
|
|
|
318
336
|
|
|
319
337
|
function persistSequence(eventsPath: string, seq: number): void {
|
|
320
338
|
try {
|
|
321
|
-
|
|
339
|
+
// P0-4: the .seq sidecar is disposable (event-reconstructor tolerates an
|
|
340
|
+
// inconsistent tail); best-effort avoids 2 fsyncs per event append.
|
|
341
|
+
atomicWriteFile(sequencePath(eventsPath), String(seq), { durability: "best-effort" });
|
|
322
342
|
} catch (error) {
|
|
323
343
|
logInternalError("event-log.persist-sequence-file", error, `eventsPath=${eventsPath}`);
|
|
324
344
|
}
|
|
@@ -501,7 +521,8 @@ async function withEventLogLockAsync<T>(
|
|
|
501
521
|
try {
|
|
502
522
|
await fs.promises.mkdir(lockDir);
|
|
503
523
|
try {
|
|
504
|
-
|
|
524
|
+
// P0-4: the lock pid file is disposable stale-lock state; best-effort.
|
|
525
|
+
atomicWriteFile(pidFile, String(process.pid), { durability: "best-effort" });
|
|
505
526
|
} catch {
|
|
506
527
|
/* best-effort */
|
|
507
528
|
}
|
|
@@ -724,12 +745,15 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
724
745
|
// Phase 1.5: when worker atomic writer is enabled, append via worker.
|
|
725
746
|
if (isWorkerAtomicWriterEnabled()) {
|
|
726
747
|
await appendFileViaWorker(eventsPath, line);
|
|
727
|
-
//
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
await
|
|
731
|
-
|
|
732
|
-
|
|
748
|
+
// P0-4 (F3a mirror): fsync terminal events only; non-terminal events are
|
|
749
|
+
// informational and the event-reconstructor tolerates an inconsistent tail.
|
|
750
|
+
if (isTerminal) {
|
|
751
|
+
const fd = await fs.promises.open(eventsPath, "r+");
|
|
752
|
+
try {
|
|
753
|
+
await fd.sync();
|
|
754
|
+
} finally {
|
|
755
|
+
await fd.close();
|
|
756
|
+
}
|
|
733
757
|
}
|
|
734
758
|
} else {
|
|
735
759
|
// FIND-10: single-fd append+fsync. Opens in append mode, writes,
|
|
@@ -740,7 +764,8 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
740
764
|
const fd = await fs.promises.open(eventsPath, "a");
|
|
741
765
|
try {
|
|
742
766
|
await fd.appendFile(line, "utf-8");
|
|
743
|
-
|
|
767
|
+
// P0-4 (F3a mirror): skip the data fsync for non-terminal events.
|
|
768
|
+
if (isTerminal) await fd.sync();
|
|
744
769
|
// FIND-10 R1 fix: the cache-optimization fd.stat() must NOT sit in the
|
|
745
770
|
// seq-durability critical path. If it threw (rare — fd invalidated),
|
|
746
771
|
// it would skip persistSequence below and reopen the seq-reuse
|
|
@@ -764,7 +789,7 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
764
789
|
// FIND-10: track whether compaction happened after the append so the
|
|
765
790
|
// cache-update stat can safely reuse postAppendStat (file unchanged).
|
|
766
791
|
let compactedAfterAppend = false;
|
|
767
|
-
if (
|
|
792
|
+
if (tickAppendCounter(eventsPath) && needsRotation(eventsPath)) {
|
|
768
793
|
compactedAfterAppend = true;
|
|
769
794
|
try {
|
|
770
795
|
const prepared = prepareCompaction(eventsPath);
|
|
@@ -865,6 +890,8 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
865
890
|
*/
|
|
866
891
|
async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedAppend[]): Promise<void> {
|
|
867
892
|
if (queue.length === 0) return;
|
|
893
|
+
// P0-2: keep the per-path counter honest for cross-path rotation sampling.
|
|
894
|
+
tickAppendCounter(eventsPath, queue.length);
|
|
868
895
|
fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
|
|
869
896
|
|
|
870
897
|
// Pre-flight size check (mirrors appendEventInsideLock). We do it once for
|
|
@@ -1100,8 +1127,7 @@ function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): Team
|
|
|
1100
1127
|
logInternalError("event-log.persist-sequence", error, `eventsPath=${eventsPath}`);
|
|
1101
1128
|
}
|
|
1102
1129
|
}
|
|
1103
|
-
|
|
1104
|
-
if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
|
|
1130
|
+
if (tickAppendCounter(eventsPath) && needsRotation(eventsPath)) {
|
|
1105
1131
|
// Round 24 (BUG 1): we are INSIDE withEventLogLockSync here (called via
|
|
1106
1132
|
// appendEventInsideLock). The mkdir lock is NOT re-entrant, so calling the
|
|
1107
1133
|
// locked compactEventLog would deadlock → 5s timeout → compaction never
|
package/src/state/state-store.ts
CHANGED
|
@@ -151,19 +151,33 @@ function scopeBaseRoot(cwd: string): string {
|
|
|
151
151
|
return useProjectState(cwd) ? projectCrewRoot(cwd) : userCrewRoot();
|
|
152
152
|
}
|
|
153
153
|
|
|
154
|
+
// P1-12: the containment verdict (resolveRealContainedPath ancestor walk, ~8-12
|
|
155
|
+
// syscalls) cannot change for a run dir that isn't replaced — memoize positive
|
|
156
|
+
// results per (cwd, runId) with a short TTL. Negatives are NOT cached (so a
|
|
157
|
+
// newly-created run is found promptly). Downstream manifest stat still catches
|
|
158
|
+
// deletions, so a stale positive is harmless.
|
|
159
|
+
const runStateRootCache = new Map<string, { root: string; expiresAt: number }>();
|
|
160
|
+
const RUN_STATE_ROOT_TTL_MS = 10_000;
|
|
161
|
+
const RUN_STATE_ROOT_CACHE_MAX = 256;
|
|
162
|
+
|
|
154
163
|
function resolveRunStateRoot(cwd: string, runId: string): string | undefined {
|
|
155
164
|
assertSafePathId("runId", runId);
|
|
165
|
+
const key = `${cwd}\0${runId}`;
|
|
166
|
+
const now = Date.now();
|
|
167
|
+
const cached = runStateRootCache.get(key);
|
|
168
|
+
if (cached && cached.expiresAt > now) return cached.root;
|
|
156
169
|
const runsRoot = path.join(scopeBaseRoot(cwd), DEFAULT_PATHS.state.runsSubdir);
|
|
157
170
|
const scopedPath = resolveContainedRelativePath(runsRoot, runId, "runId");
|
|
158
171
|
try {
|
|
159
|
-
// Single atomic validation: resolves through symlinks via realpath,
|
|
160
|
-
// verifies containment within runsRoot, and throws ENOENT if missing.
|
|
161
|
-
// Eliminates the TOCTOU window from the previous existsSync + lstatSync
|
|
162
|
-
// + resolveRealContainedPath sequence.
|
|
163
172
|
resolveRealContainedPath(runsRoot, runId);
|
|
164
173
|
} catch {
|
|
165
174
|
return undefined;
|
|
166
175
|
}
|
|
176
|
+
if (runStateRootCache.size >= RUN_STATE_ROOT_CACHE_MAX) {
|
|
177
|
+
const oldest = runStateRootCache.keys().next().value;
|
|
178
|
+
if (oldest !== undefined) runStateRootCache.delete(oldest);
|
|
179
|
+
}
|
|
180
|
+
runStateRootCache.set(key, { root: scopedPath, expiresAt: now + RUN_STATE_ROOT_TTL_MS });
|
|
167
181
|
return scopedPath;
|
|
168
182
|
}
|
|
169
183
|
|
package/src/utils/redaction.ts
CHANGED
|
@@ -222,7 +222,46 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
222
222
|
return true;
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
+
// P1-6: cheap pre-filter. The common clean case (plain output, JSON event
|
|
226
|
+
// lines without secret-like key names) bails after one sweep instead of running
|
|
227
|
+
// all ~14 redaction passes. The marker set is the NECESSARY substring for every
|
|
228
|
+
// redactable pattern below — a string containing none of them cannot hold a
|
|
229
|
+
// redactable secret, so skipping is safe. Any marker present → full redaction.
|
|
230
|
+
// NOTE: keep this set in sync with the patterns in redactSecretString; the
|
|
231
|
+
// redaction-corpus test asserts every known secret type trips the pre-filter.
|
|
232
|
+
const SECRET_MARKERS_CASE_SENSITIVE = [
|
|
233
|
+
"-----BEGIN",
|
|
234
|
+
"eyJ",
|
|
235
|
+
"AKIA",
|
|
236
|
+
"AIza",
|
|
237
|
+
"sk_live_",
|
|
238
|
+
"xox",
|
|
239
|
+
// GitHub PAT prefixes (gh[pousr]_) — exact, not the over-broad "gh" which
|
|
240
|
+
// matched "though"/"highlight"/"might" and defeated the pre-filter skip.
|
|
241
|
+
"ghp_",
|
|
242
|
+
"gho_",
|
|
243
|
+
"ghu_",
|
|
244
|
+
"ghs_",
|
|
245
|
+
"ghr_",
|
|
246
|
+
];
|
|
247
|
+
const SECRET_MARKERS_LOWER = ["bearer", "authorization", "token", "api", "key", "password", "passwd", "secret", "credential", "private"];
|
|
248
|
+
|
|
249
|
+
function mayContainSecret(value: string): boolean {
|
|
250
|
+
for (const m of SECRET_MARKERS_CASE_SENSITIVE) {
|
|
251
|
+
if (value.includes(m)) return true;
|
|
252
|
+
}
|
|
253
|
+
// Case-insensitive markers (Bearer/auth headers + inline-secret key names).
|
|
254
|
+
// toLowerCase is a full pass, so only pay it after the case-sensitive misses.
|
|
255
|
+
const lower = value.toLowerCase();
|
|
256
|
+
for (const m of SECRET_MARKERS_LOWER) {
|
|
257
|
+
if (lower.includes(m)) return true;
|
|
258
|
+
}
|
|
259
|
+
return false;
|
|
260
|
+
}
|
|
261
|
+
|
|
225
262
|
export function redactSecretString(value: string): string {
|
|
263
|
+
// P1-6: skip the ~14 redaction passes when no secret marker is present.
|
|
264
|
+
if (!mayContainSecret(value)) return value;
|
|
226
265
|
let result = value;
|
|
227
266
|
|
|
228
267
|
// Replace PEM private keys. Skip the regex entirely on very large inputs —
|
|
@@ -463,15 +463,30 @@ async function branchExistsAsync(repoRoot: string, branch: string): Promise<{ lo
|
|
|
463
463
|
}
|
|
464
464
|
}
|
|
465
465
|
|
|
466
|
+
// P1-11: `git worktree prune` is a repo-level write that cleans ALL stale
|
|
467
|
+
// worktrees — running it per task is pure waste. Coalesce concurrent calls
|
|
468
|
+
// (N tasks in a batch → 1 prune) and memoize per repo per process (within a
|
|
469
|
+
// run no new stale worktrees appear, so one prune suffices).
|
|
470
|
+
const _prunedRepos = new Set<string>();
|
|
471
|
+
const _pruneInFlight = new Map<string, Promise<void>>();
|
|
466
472
|
async function pruneStaleWorktreesAsync(repoRoot: string): Promise<void> {
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
473
|
+
if (_prunedRepos.has(repoRoot)) return;
|
|
474
|
+
const inFlight = _pruneInFlight.get(repoRoot);
|
|
475
|
+
if (inFlight) return inFlight;
|
|
476
|
+
const p = (async () => {
|
|
477
|
+
try {
|
|
478
|
+
await execFileAsync("git", ["worktree", "prune"], {
|
|
479
|
+
cwd: repoRoot,
|
|
480
|
+
windowsHide: true,
|
|
481
|
+
});
|
|
482
|
+
} catch {
|
|
483
|
+
/* best-effort */
|
|
484
|
+
}
|
|
485
|
+
_prunedRepos.add(repoRoot);
|
|
486
|
+
_pruneInFlight.delete(repoRoot);
|
|
487
|
+
})();
|
|
488
|
+
_pruneInFlight.set(repoRoot, p);
|
|
489
|
+
return p;
|
|
475
490
|
}
|
|
476
491
|
|
|
477
492
|
/**
|