pi-subagents 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -19
- package/agents/oracle.md +7 -5
- package/agents/researcher.md +1 -1
- package/agents/reviewer.md +2 -2
- package/agents/scout.md +1 -1
- package/agents/worker.md +1 -1
- package/docs/agents.md +3 -0
- package/docs/configuration.md +75 -8
- package/docs/extension-api.md +36 -0
- package/docs/missions.md +5 -3
- package/docs/observability.md +42 -2
- package/docs/tool-reference.md +19 -2
- package/docs/workflows.md +2 -2
- package/package.json +1 -1
- package/skills/pi-subagents/references/constraints-and-recipes.md +3 -2
- package/skills/pi-subagents/references/execution-controls.md +5 -4
- package/skills/pi-subagents/references/management-authoring-rpc.md +2 -2
- package/skills/pi-subagents/references/prompting-and-roles.md +31 -15
- package/src/agents/agent-serializer.ts +2 -0
- package/src/agents/agents.ts +37 -12
- package/src/api/external-runs.ts +174 -84
- package/src/api/preflight.ts +13 -7
- package/src/extension/config.ts +53 -0
- package/src/extension/index.ts +62 -9
- package/src/extension/public-execution.ts +32 -4
- package/src/extension/rpc.ts +5 -1
- package/src/extension/schemas.ts +10 -8
- package/src/extension/tool-description.ts +13 -9
- package/src/inspectors/herdr/actions.ts +13 -8
- package/src/inspectors/herdr/inspector-runner.ts +16 -3
- package/src/inspectors/herdr/project-panes.ts +2 -6
- package/src/inspectors/herdr/shell-command.ts +16 -0
- package/src/intercom/intercom-bridge.ts +5 -4
- package/src/intercom/native-supervisor-channel.ts +19 -42
- package/src/missions/goal-driver.ts +3 -1
- package/src/missions/store.ts +8 -3
- package/src/runs/background/active-async-capacity.ts +82 -25
- package/src/runs/background/active-run-index.ts +71 -1
- package/src/runs/background/async-execution.ts +73 -43
- package/src/runs/background/async-job-tracker.ts +5 -0
- package/src/runs/background/async-resume.ts +14 -6
- package/src/runs/background/async-status-snapshot.ts +277 -0
- package/src/runs/background/async-status.ts +8 -3
- package/src/runs/background/chain-root-attachment.ts +2 -2
- package/src/runs/background/completion-replay.ts +11 -1
- package/src/runs/background/fleet-view.ts +21 -6
- package/src/runs/background/result-files.ts +437 -0
- package/src/runs/background/result-watcher.ts +188 -41
- package/src/runs/background/resume-guidance.ts +27 -7
- package/src/runs/background/retained-children.ts +75 -18
- package/src/runs/background/run-id-resolver.ts +30 -24
- package/src/runs/background/run-status.ts +101 -4
- package/src/runs/background/scheduled-runs.ts +54 -28
- package/src/runs/background/stale-run-reconciler.ts +27 -13
- package/src/runs/background/subagent-runner.ts +298 -33
- package/src/runs/background/subagent-wait.ts +2 -0
- package/src/runs/background/wait-completions.ts +5 -2
- package/src/runs/foreground/async-dismiss-action.ts +2 -1
- package/src/runs/foreground/chain-execution.ts +16 -0
- package/src/runs/foreground/execution.ts +219 -15
- package/src/runs/foreground/prompt-audit.ts +4 -3
- package/src/runs/foreground/subagent-executor.ts +328 -43
- package/src/runs/shared/completion-guard.ts +107 -1
- package/src/runs/shared/external-cli-runner.ts +4 -0
- package/src/runs/shared/llm-intent-arbiter.ts +39 -23
- package/src/runs/shared/model-fallback.ts +16 -2
- package/src/runs/shared/nested-events.ts +66 -62
- package/src/runs/shared/orca-progress-tabs.ts +375 -0
- package/src/runs/shared/parallel-utils.ts +2 -0
- package/src/runs/shared/subagent-control.ts +15 -0
- package/src/runs/shared/subagent-prompt-runtime.ts +1 -9
- package/src/runs/shared/subagent-startup-retry.ts +12 -0
- package/src/runs/shared/tool-timeout.ts +93 -0
- package/src/shared/agent-stream-options.ts +5 -0
- package/src/shared/artifacts.ts +2 -6
- package/src/shared/display-text.ts +50 -0
- package/src/shared/node-executable.ts +21 -0
- package/src/shared/types.ts +51 -5
- package/src/slash/slash-commands.ts +34 -25
- package/src/slash/slash-live-state.ts +3 -0
- package/src/tui/fleet-status.ts +160 -45
- package/src/tui/fleet-transcript.ts +1 -48
- package/src/tui/fleet.ts +128 -16
- package/src/tui/render.ts +122 -44
- package/src/watchdog/permission-arbiter.ts +2 -1
- package/src/watchdog/review.ts +4 -3
- package/src/workflows/chat-progress.ts +10 -2
- package/src/workflows/scripted-workflow.ts +272 -76
|
@@ -12,7 +12,7 @@ import { flatToLogicalStepIndex, normalizeParallelGroups } from "./parallel-grou
|
|
|
12
12
|
import { contextModeLabel, summarizeContextModes, type ContextMode, type ContextSummary } from "../shared/context-mode.ts";
|
|
13
13
|
import { reconcileAsyncRun, reconcileNestedAsyncDescendants } from "./stale-run-reconciler.ts";
|
|
14
14
|
import { readProcessTerminal, sanitizeProcessTerminal } from "./process-terminal.ts";
|
|
15
|
-
import { ACTIVE_RUN_INDEX_DIR, isActiveAsyncState, readActiveRunIndex, releaseActiveRunIndex, updateActiveRunIndex } from "./active-run-index.ts";
|
|
15
|
+
import { ACTIVE_RUN_INDEX_DIR, DEFAULT_STALE_TERMINAL_ACTIVE_MARKER_MS, activeRunMarkerAgeMs, isActiveAsyncState, readActiveRunIndex, releaseActiveRunIndex, updateActiveRunIndex } from "./active-run-index.ts";
|
|
16
16
|
|
|
17
17
|
interface AsyncRunStepSummary {
|
|
18
18
|
index: number;
|
|
@@ -25,6 +25,7 @@ interface AsyncRunStepSummary {
|
|
|
25
25
|
structured?: boolean;
|
|
26
26
|
checkpoint?: ChainCheckpointState;
|
|
27
27
|
status: AsyncJobStep["status"];
|
|
28
|
+
runner?: AsyncJobStep["runner"];
|
|
28
29
|
activityState?: ActivityState;
|
|
29
30
|
lastActivityAt?: number;
|
|
30
31
|
currentTool?: string;
|
|
@@ -128,6 +129,7 @@ interface AsyncRunListOptions {
|
|
|
128
129
|
now?: () => number;
|
|
129
130
|
reconcile?: boolean;
|
|
130
131
|
runId?: string;
|
|
132
|
+
includeNested?: boolean;
|
|
131
133
|
}
|
|
132
134
|
|
|
133
135
|
function getErrorMessage(error: unknown): string {
|
|
@@ -253,6 +255,7 @@ function statusToSummary(asyncDir: string, status: AsyncStatus & { cwd?: string
|
|
|
253
255
|
...(step.structured ? { structured: step.structured } : {}),
|
|
254
256
|
...(step.checkpoint ? { checkpoint: step.checkpoint } : {}),
|
|
255
257
|
status: step.status,
|
|
258
|
+
...(step.runner ? { runner: step.runner } : {}),
|
|
256
259
|
...(stepActivityState ? { activityState: stepActivityState } : {}),
|
|
257
260
|
...(stepLastActivityAt ? { lastActivityAt: stepLastActivityAt } : {}),
|
|
258
261
|
...(step.currentTool ? { currentTool: step.currentTool } : {}),
|
|
@@ -383,6 +386,7 @@ export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions
|
|
|
383
386
|
&& options.states !== undefined
|
|
384
387
|
&& options.states.length > 0
|
|
385
388
|
&& options.states.every(isActiveAsyncState);
|
|
389
|
+
const includeNested = options.includeNested !== false;
|
|
386
390
|
try {
|
|
387
391
|
if (options.runId !== undefined) {
|
|
388
392
|
const resolution = resolveTargetedAsyncRun(asyncDirRoot, options.runId, options.sessionId);
|
|
@@ -443,6 +447,7 @@ export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions
|
|
|
443
447
|
// entirely when no active runs match.
|
|
444
448
|
let nestedRouteIndex: Map<string, NestedRoute> | undefined;
|
|
445
449
|
const resolveNestedRoute = (rootRunId: string): NestedRoute | undefined => {
|
|
450
|
+
if (!includeNested) return undefined;
|
|
446
451
|
if (usedActiveIndex) return findNestedRouteForRootId(rootRunId);
|
|
447
452
|
if (!nestedRouteIndex) nestedRouteIndex = buildNestedRouteIndex();
|
|
448
453
|
return nestedRouteIndex.get(rootRunId);
|
|
@@ -459,7 +464,7 @@ export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions
|
|
|
459
464
|
}
|
|
460
465
|
if (usedActiveIndex && !isActiveAsyncState(status.state)) {
|
|
461
466
|
const processTerminal = readProcessTerminal(asyncDir, { runId: status.runId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId });
|
|
462
|
-
if (processTerminal?.state === "observed") releaseActiveRunIndex(asyncDir);
|
|
467
|
+
if (processTerminal?.state === "observed" || (activeRunMarkerAgeMs(asyncDir, options.now?.()) ?? 0) > DEFAULT_STALE_TERMINAL_ACTIVE_MARKER_MS) releaseActiveRunIndex(asyncDir);
|
|
463
468
|
}
|
|
464
469
|
if (status.displayDismissedAt !== undefined) continue;
|
|
465
470
|
// Filter before the nested-route lookup: the lookup builds an index over
|
|
@@ -470,7 +475,7 @@ export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions
|
|
|
470
475
|
if (options.sessionId && status.sessionId !== options.sessionId) continue;
|
|
471
476
|
const nestedWarnings: string[] = [];
|
|
472
477
|
let nestedRoute: NestedRoute | undefined;
|
|
473
|
-
if (options.reconcile !== false) {
|
|
478
|
+
if (options.reconcile !== false && includeNested) {
|
|
474
479
|
try {
|
|
475
480
|
nestedRoute = resolveNestedRoute(status.runId || path.basename(asyncDir));
|
|
476
481
|
if (nestedRoute) reconcileNestedAsyncDescendants(nestedRoute, { resultsDir: options.resultsDir, kill: options.kill, now: options.now });
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
|
-
import
|
|
2
|
+
import { resultFilePath } from "./result-files.ts";
|
|
3
3
|
import type { AcceptanceLedger, AsyncStatus, CostSummary, ModelAttempt } from "../../shared/types.ts";
|
|
4
4
|
import { readStatus } from "../../shared/utils.ts";
|
|
5
5
|
|
|
@@ -195,5 +195,5 @@ export async function waitForImportedAsyncRoot(
|
|
|
195
195
|
}
|
|
196
196
|
|
|
197
197
|
export function resolveAsyncRootResultPath(resultsDir: string, runId: string): string {
|
|
198
|
-
return
|
|
198
|
+
return resultFilePath(resultsDir, runId);
|
|
199
199
|
}
|
|
@@ -9,6 +9,8 @@ const ARCHIVE_VERSION = 1;
|
|
|
9
9
|
const ARCHIVE_TEXT_LIMIT_BYTES = 64 * 1024;
|
|
10
10
|
const REPLAY_DIR_NAME = "completion-replay";
|
|
11
11
|
const ARCHIVE_DIR_NAME = "output-archives";
|
|
12
|
+
const CLEANUP_INTERVAL_MS = 60_000;
|
|
13
|
+
const lastCleanupByResultsDir = new Map<string, number>();
|
|
12
14
|
|
|
13
15
|
export interface CompletionArchiveEntry {
|
|
14
16
|
agent?: string;
|
|
@@ -194,7 +196,7 @@ export function writeCompletionReplay(input: {
|
|
|
194
196
|
archivePath,
|
|
195
197
|
};
|
|
196
198
|
writePrivateAtomicJson(completionReplayPath(input.resultsDir, input.runId), record);
|
|
197
|
-
|
|
199
|
+
cleanupCompletionReplayIfDue(input.resultsDir, input.now, input.ttlMs);
|
|
198
200
|
return record;
|
|
199
201
|
}
|
|
200
202
|
|
|
@@ -233,6 +235,14 @@ export function readCompletionArchive(archivePath: string): CompletionArchive |
|
|
|
233
235
|
}
|
|
234
236
|
}
|
|
235
237
|
|
|
238
|
+
export function cleanupCompletionReplayIfDue(resultsDir: string, now: number, maxAgeMs: number, intervalMs = CLEANUP_INTERVAL_MS): boolean {
|
|
239
|
+
const last = lastCleanupByResultsDir.get(resultsDir);
|
|
240
|
+
if (last !== undefined && now - last < intervalMs) return false;
|
|
241
|
+
lastCleanupByResultsDir.set(resultsDir, now);
|
|
242
|
+
cleanupCompletionReplay(resultsDir, now, maxAgeMs);
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
|
|
236
246
|
/** Opportunistically remove expired replay and orphan archive files without affecting delivery. */
|
|
237
247
|
export function cleanupCompletionReplay(resultsDir: string, now: number, maxAgeMs: number): void {
|
|
238
248
|
const replayDir = path.join(resultsDir, REPLAY_DIR_NAME);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import type { AgentToolResult } from "@earendil-works/pi-agent-core";
|
|
4
|
+
import { safeTerminalText } from "../../shared/display-text.ts";
|
|
4
5
|
import { formatDuration, formatModelThinking, formatTokens, shortenPath } from "../../shared/formatters.ts";
|
|
5
6
|
import { formatActivityLabel } from "../../shared/status-format.ts";
|
|
6
7
|
import {
|
|
@@ -196,7 +197,7 @@ function readSessionTranscriptTail(sessionFile: string, maxLines: number, truste
|
|
|
196
197
|
try {
|
|
197
198
|
const parsed = JSON.parse(line) as unknown;
|
|
198
199
|
const messageLine = sessionMessageLine(parsed);
|
|
199
|
-
if (messageLine) lines.push(messageLine);
|
|
200
|
+
if (messageLine) lines.push(...messageLine.split(/\r?\n/));
|
|
200
201
|
} catch {
|
|
201
202
|
malformed++;
|
|
202
203
|
}
|
|
@@ -410,6 +411,18 @@ function appendKnownArtifacts(lines: string[], input: { outputPaths: string[]; s
|
|
|
410
411
|
for (const artifact of artifacts) lines.push(` ${artifact}`);
|
|
411
412
|
}
|
|
412
413
|
|
|
414
|
+
/**
|
|
415
|
+
* Sanitizes each assembled line independently.
|
|
416
|
+
*
|
|
417
|
+
* Sanitizing the joined response would let a single binary fragment in child
|
|
418
|
+
* output collapse the whole transcript, including run state, artifact paths, and
|
|
419
|
+
* warnings, into the binary placeholder. Per-line sanitization keeps that damage
|
|
420
|
+
* to the offending line.
|
|
421
|
+
*/
|
|
422
|
+
function safeTranscriptLines(lines: string[]): string {
|
|
423
|
+
return lines.map((line) => safeTerminalText(line)).join("\n");
|
|
424
|
+
}
|
|
425
|
+
|
|
413
426
|
function appendTranscriptBody(lines: string[], sourceLabel: string, sourceLines: string[], truncated: boolean): void {
|
|
414
427
|
lines.push(`${sourceLabel}${truncated ? " (tail truncated)" : ""}:`);
|
|
415
428
|
if (sourceLines.length === 0) {
|
|
@@ -430,6 +443,8 @@ export function formatAsyncRunTranscript(status: AsyncStatus, asyncDir: string,
|
|
|
430
443
|
: uniqueStrings([runOutputPath]);
|
|
431
444
|
const sessionFile = selected.index !== undefined ? selected.step?.sessionFile : status.sessionFile;
|
|
432
445
|
const eventsPath = path.join(asyncDir, "events.jsonl");
|
|
446
|
+
// Synthesized output paths can name files that were never written. Keep the unfiltered paths
|
|
447
|
+
// below so a present-but-unreadable file still reports its read error.
|
|
433
448
|
|
|
434
449
|
const context = contextModeLabel(status.context ?? summarizeContextModes((status.steps ?? []).map((step) => step.context)));
|
|
435
450
|
const lines = [
|
|
@@ -439,7 +454,7 @@ export function formatAsyncRunTranscript(status: AsyncStatus, asyncDir: string,
|
|
|
439
454
|
stepStateLine(status.mode, selected.index, selected.step),
|
|
440
455
|
selected.hint,
|
|
441
456
|
].filter((line): line is string => Boolean(line));
|
|
442
|
-
appendKnownArtifacts(lines, { outputPaths, sessionFile, eventsPath: fs.existsSync(eventsPath) ? eventsPath : undefined, logPath: fs.existsSync(logPath) ? logPath : undefined });
|
|
457
|
+
appendKnownArtifacts(lines, { outputPaths: outputPaths.filter((outputPath) => fs.existsSync(outputPath)), sessionFile, eventsPath: fs.existsSync(eventsPath) ? eventsPath : undefined, logPath: fs.existsSync(logPath) ? logPath : undefined });
|
|
443
458
|
|
|
444
459
|
const warnings: string[] = [];
|
|
445
460
|
let transcriptLines: string[] = [];
|
|
@@ -470,7 +485,7 @@ export function formatAsyncRunTranscript(status: AsyncStatus, asyncDir: string,
|
|
|
470
485
|
for (const warning of warnings) lines.push(` ${warning}`);
|
|
471
486
|
}
|
|
472
487
|
appendTranscriptBody(lines, transcriptSource, transcriptLines, truncated);
|
|
473
|
-
return lines
|
|
488
|
+
return safeTranscriptLines(lines);
|
|
474
489
|
}
|
|
475
490
|
|
|
476
491
|
export function formatNestedRunTranscript(run: NestedRunSummary, options: TranscriptOptions = {}): string {
|
|
@@ -488,7 +503,7 @@ export function formatNestedRunTranscript(run: NestedRunSummary, options: Transc
|
|
|
488
503
|
appendKnownArtifacts(lines, { outputPaths: [], sessionFile: run.sessionFile });
|
|
489
504
|
if (!run.sessionFile) {
|
|
490
505
|
appendTranscriptBody(lines, "Transcript tail", [], false);
|
|
491
|
-
return lines
|
|
506
|
+
return safeTranscriptLines(lines);
|
|
492
507
|
}
|
|
493
508
|
const sessionTail = readSessionTranscriptTail(run.sessionFile, lineLimit, options.sessionRoots ?? []);
|
|
494
509
|
if (sessionTail.warnings.length) {
|
|
@@ -496,7 +511,7 @@ export function formatNestedRunTranscript(run: NestedRunSummary, options: Transc
|
|
|
496
511
|
for (const warning of sessionTail.warnings) lines.push(` ${warning}`);
|
|
497
512
|
}
|
|
498
513
|
appendTranscriptBody(lines, `Session transcript tail from ${run.sessionFile}`, sessionTail.lines, false);
|
|
499
|
-
return lines
|
|
514
|
+
return safeTranscriptLines(lines);
|
|
500
515
|
}
|
|
501
516
|
|
|
502
517
|
export function formatAsyncResultTranscript(data: {
|
|
@@ -536,5 +551,5 @@ export function formatAsyncResultTranscript(data: {
|
|
|
536
551
|
].filter((line): line is string => Boolean(line));
|
|
537
552
|
appendKnownArtifacts(lines, { outputPaths: [], sessionFile, resultPath });
|
|
538
553
|
appendTranscriptBody(lines, "Result transcript tail", transcriptLines.filter((line) => line.trim()), output.split(/\r?\n/).length > lineLimit);
|
|
539
|
-
return lines
|
|
554
|
+
return safeTranscriptLines(lines);
|
|
540
555
|
}
|
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { writeAtomicJson } from "../../shared/atomic-json.ts";
|
|
4
|
+
import { MISSION_BINDING_FILE } from "../../missions/lifecycle.ts";
|
|
5
|
+
|
|
6
|
+
const RESULT_INDEX_VERSION = 1;
|
|
7
|
+
const RESULT_INDEX_DIR = "result-index";
|
|
8
|
+
const SESSION_INDEX_DIR = "sessions";
|
|
9
|
+
const OBSERVER_INDEX_DIR = "observers";
|
|
10
|
+
const TOOL_CALL_INDEX_DIR = "tool-calls";
|
|
11
|
+
const RESULT_PENDING_DIR = "result-pending";
|
|
12
|
+
const MISSION_OBSERVER = "mission";
|
|
13
|
+
|
|
14
|
+
export interface ResultIndexEntry {
|
|
15
|
+
version: 1;
|
|
16
|
+
runId: string;
|
|
17
|
+
sessionId: string;
|
|
18
|
+
file: string;
|
|
19
|
+
writtenAt: number;
|
|
20
|
+
asyncDir?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function encodeSegment(value: string): string {
|
|
24
|
+
return encodeURIComponent(value);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function nonEmptyString(value: unknown): string | undefined {
|
|
28
|
+
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function resultFileName(runId: string): string {
|
|
32
|
+
return `${runId}.json`;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function resultFilePath(resultsDir: string, runId: string): string {
|
|
36
|
+
return path.join(resultsDir, resultFileName(runId));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function sessionIndexDir(resultsDir: string, sessionId: string): string {
|
|
40
|
+
return path.join(resultsDir, RESULT_INDEX_DIR, SESSION_INDEX_DIR, encodeSegment(sessionId));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function resultIndexPath(resultsDir: string, sessionId: string, runId: string): string {
|
|
44
|
+
return path.join(sessionIndexDir(resultsDir, sessionId), `${encodeSegment(runId)}.json`);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function resultPendingPath(resultsDir: string, sessionId: string, runId: string): string {
|
|
48
|
+
return path.join(resultsDir, RESULT_PENDING_DIR, encodeSegment(sessionId), `${encodeSegment(runId)}.json`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function observerIndexDir(resultsDir: string, observer: string): string {
|
|
52
|
+
return path.join(resultsDir, RESULT_INDEX_DIR, OBSERVER_INDEX_DIR, observer);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function observerIndexPath(resultsDir: string, observer: string, runId: string): string {
|
|
56
|
+
return path.join(observerIndexDir(resultsDir, observer), `${encodeSegment(runId)}.json`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function toolCallIndexDir(resultsDir: string, toolCallId: string): string {
|
|
60
|
+
return path.join(resultsDir, RESULT_INDEX_DIR, TOOL_CALL_INDEX_DIR, encodeSegment(toolCallId));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function toolCallIndexPath(resultsDir: string, toolCallId: string, runId: string): string {
|
|
64
|
+
return path.join(toolCallIndexDir(resultsDir, toolCallId), `${encodeSegment(runId)}.json`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function parseResultIndexEntry(value: unknown): ResultIndexEntry | undefined {
|
|
68
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
69
|
+
const record = value as Partial<ResultIndexEntry>;
|
|
70
|
+
if (record.version !== RESULT_INDEX_VERSION
|
|
71
|
+
|| typeof record.runId !== "string"
|
|
72
|
+
|| typeof record.sessionId !== "string"
|
|
73
|
+
|| typeof record.file !== "string"
|
|
74
|
+
|| typeof record.writtenAt !== "number") return undefined;
|
|
75
|
+
return {
|
|
76
|
+
version: RESULT_INDEX_VERSION,
|
|
77
|
+
runId: record.runId,
|
|
78
|
+
sessionId: record.sessionId,
|
|
79
|
+
file: record.file,
|
|
80
|
+
writtenAt: record.writtenAt,
|
|
81
|
+
...(typeof record.asyncDir === "string" ? { asyncDir: record.asyncDir } : {}),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function writeResultIndexForData(resultPath: string, data: Record<string, unknown>): void {
|
|
86
|
+
const runId = nonEmptyString(data.runId) ?? nonEmptyString(data.id) ?? path.basename(resultPath, ".json");
|
|
87
|
+
const sessionId = nonEmptyString(data.sessionId);
|
|
88
|
+
if (!runId || !sessionId) return;
|
|
89
|
+
const file = path.basename(resultPath);
|
|
90
|
+
const entry: ResultIndexEntry = {
|
|
91
|
+
version: RESULT_INDEX_VERSION,
|
|
92
|
+
runId,
|
|
93
|
+
sessionId,
|
|
94
|
+
file,
|
|
95
|
+
writtenAt: Date.now(),
|
|
96
|
+
...(nonEmptyString(data.asyncDir) ? { asyncDir: nonEmptyString(data.asyncDir)! } : {}),
|
|
97
|
+
};
|
|
98
|
+
const resultsDir = path.dirname(resultPath);
|
|
99
|
+
writeAtomicJson(resultIndexPath(resultsDir, sessionId, runId), entry);
|
|
100
|
+
const toolCallId = nonEmptyString(data.toolCallId);
|
|
101
|
+
try {
|
|
102
|
+
if (toolCallId) writeAtomicJson(toolCallIndexPath(resultsDir, toolCallId, runId), entry);
|
|
103
|
+
} catch (error) {
|
|
104
|
+
console.error(`Failed to write async result tool-call index for '${resultPath}':`, error);
|
|
105
|
+
}
|
|
106
|
+
try {
|
|
107
|
+
if (entry.asyncDir && fs.existsSync(path.join(entry.asyncDir, MISSION_BINDING_FILE))) {
|
|
108
|
+
writeAtomicJson(observerIndexPath(resultsDir, MISSION_OBSERVER, runId), entry);
|
|
109
|
+
}
|
|
110
|
+
} catch (error) {
|
|
111
|
+
console.error(`Failed to write async result observer index for '${resultPath}':`, error);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function writeIndexedPendingResultFile(resultPath: string, data: Record<string, unknown>): { runId: string; sessionId: string; resultsDir: string } {
|
|
116
|
+
const runId = nonEmptyString(data.runId) ?? nonEmptyString(data.id) ?? path.basename(resultPath, ".json");
|
|
117
|
+
const sessionId = nonEmptyString(data.sessionId);
|
|
118
|
+
if (!sessionId) throw new Error(`Cannot write async result '${resultPath}' without a sessionId.`);
|
|
119
|
+
const resultsDir = path.dirname(resultPath);
|
|
120
|
+
writeAtomicJson(resultPendingPath(resultsDir, sessionId, runId), data);
|
|
121
|
+
writeResultIndexForData(resultPath, data);
|
|
122
|
+
return { runId, sessionId, resultsDir };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function writePendingAsyncResultFile(resultPath: string, data: Record<string, unknown>): void {
|
|
126
|
+
writeIndexedPendingResultFile(resultPath, data);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function writeAsyncResultFile(resultPath: string, data: Record<string, unknown>): { state: "public" | "pending" } {
|
|
130
|
+
const { runId, sessionId, resultsDir } = writeIndexedPendingResultFile(resultPath, data);
|
|
131
|
+
return promotePendingResultFile(resultsDir, sessionId, runId, path.basename(resultPath), { logFailure: false }) === "promoted"
|
|
132
|
+
? { state: "public" }
|
|
133
|
+
: { state: "pending" };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function removeResultIndex(resultsDir: string, sessionId: string | undefined, runId: string | undefined, toolCallId?: string): void {
|
|
137
|
+
if (!runId) return;
|
|
138
|
+
if (sessionId) {
|
|
139
|
+
try {
|
|
140
|
+
fs.rmSync(resultIndexPath(resultsDir, sessionId, runId), { force: true });
|
|
141
|
+
} catch {
|
|
142
|
+
// Index cleanup must not affect result delivery.
|
|
143
|
+
}
|
|
144
|
+
try {
|
|
145
|
+
fs.rmSync(resultPendingPath(resultsDir, sessionId, runId), { force: true });
|
|
146
|
+
} catch {
|
|
147
|
+
// Pending cleanup must not affect result delivery.
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (toolCallId) {
|
|
151
|
+
try {
|
|
152
|
+
fs.rmSync(toolCallIndexPath(resultsDir, toolCallId, runId), { force: true });
|
|
153
|
+
} catch {
|
|
154
|
+
// Index cleanup must not affect result delivery.
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
try {
|
|
158
|
+
fs.rmSync(observerIndexPath(resultsDir, MISSION_OBSERVER, runId), { force: true });
|
|
159
|
+
} catch {
|
|
160
|
+
// Index cleanup must not affect result delivery.
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export function removeMissionObserverIndex(resultsDir: string, runId: string | undefined): void {
|
|
165
|
+
if (!runId) return;
|
|
166
|
+
try {
|
|
167
|
+
fs.rmSync(observerIndexPath(resultsDir, MISSION_OBSERVER, runId), { force: true });
|
|
168
|
+
} catch {
|
|
169
|
+
// Observer index cleanup must not affect result delivery.
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function existingResultFile(resultPath: string): boolean {
|
|
174
|
+
try {
|
|
175
|
+
return fs.statSync(resultPath).isFile();
|
|
176
|
+
} catch (error) {
|
|
177
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Failed to inspect async result payload '${resultPath}':`, error);
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function pendingResultExists(resultsDir: string, sessionId: string, runId: string): boolean {
|
|
183
|
+
return existingResultFile(resultPendingPath(resultsDir, sessionId, runId));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function pendingResultPayloadMatches(filePath: string, sessionId: string, runId: string): boolean {
|
|
187
|
+
try {
|
|
188
|
+
const data = JSON.parse(fs.readFileSync(filePath, "utf-8")) as Record<string, unknown>;
|
|
189
|
+
return (nonEmptyString(data.runId) ?? nonEmptyString(data.id)) === runId && nonEmptyString(data.sessionId) === sessionId;
|
|
190
|
+
} catch (error) {
|
|
191
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Ignoring invalid pending async result '${filePath}':`, error);
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function promotePendingResultFile(resultsDir: string, sessionId: string, runId: string, file = resultFileName(runId), options: { logFailure?: boolean } = {}): "none" | "promoted" | "pending" {
|
|
197
|
+
if (file !== path.basename(file) || !file.endsWith(".json")) return "none";
|
|
198
|
+
const pendingPath = resultPendingPath(resultsDir, sessionId, runId);
|
|
199
|
+
if (!existingResultFile(pendingPath)) return "none";
|
|
200
|
+
const resultPath = path.join(resultsDir, file);
|
|
201
|
+
try {
|
|
202
|
+
fs.rmSync(resultPath, { force: true });
|
|
203
|
+
fs.renameSync(pendingPath, resultPath);
|
|
204
|
+
return existingResultFile(resultPath) ? "promoted" : "pending";
|
|
205
|
+
} catch (error) {
|
|
206
|
+
if (options.logFailure !== false) console.error(`Failed to promote pending async result '${pendingPath}' to '${resultPath}':`, error);
|
|
207
|
+
return "pending";
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
interface ResultPayloadLocation {
|
|
212
|
+
file: string;
|
|
213
|
+
path: string;
|
|
214
|
+
state: "public" | "pending";
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function pendingResultLocationForSessionRun(resultsDir: string, sessionId: string, runId: string, file = resultFileName(runId)): ResultPayloadLocation | undefined {
|
|
218
|
+
if (file !== path.basename(file) || !file.endsWith(".json")) return undefined;
|
|
219
|
+
const pendingPath = resultPendingPath(resultsDir, sessionId, runId);
|
|
220
|
+
return existingResultFile(pendingPath) && pendingResultPayloadMatches(pendingPath, sessionId, runId)
|
|
221
|
+
? { file, path: pendingPath, state: "pending" }
|
|
222
|
+
: undefined;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function pendingResultLocationForIndexedRun(resultsDir: string, runId: string): ResultPayloadLocation | undefined {
|
|
226
|
+
const root = path.join(resultsDir, RESULT_PENDING_DIR);
|
|
227
|
+
let sessions: fs.Dirent[];
|
|
228
|
+
try {
|
|
229
|
+
sessions = fs.readdirSync(root, { withFileTypes: true });
|
|
230
|
+
} catch (error) {
|
|
231
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Failed to inspect pending async result root '${root}':`, error);
|
|
232
|
+
return undefined;
|
|
233
|
+
}
|
|
234
|
+
for (const session of sessions) {
|
|
235
|
+
if (!session.isDirectory()) continue;
|
|
236
|
+
let sessionId: string;
|
|
237
|
+
try {
|
|
238
|
+
sessionId = decodeURIComponent(session.name);
|
|
239
|
+
} catch {
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
const location = pendingResultLocationForSessionRun(resultsDir, sessionId, runId);
|
|
243
|
+
if (location) return location;
|
|
244
|
+
}
|
|
245
|
+
return undefined;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function resultPayloadLocationFromIndex(resultsDir: string, entry: ResultIndexEntry): ResultPayloadLocation | undefined {
|
|
249
|
+
if (entry.file !== path.basename(entry.file) || !entry.file.endsWith(".json")) return undefined;
|
|
250
|
+
const pendingState = promotePendingResultFile(resultsDir, entry.sessionId, entry.runId, entry.file);
|
|
251
|
+
if (pendingState === "pending") {
|
|
252
|
+
return pendingResultLocationForSessionRun(resultsDir, entry.sessionId, entry.runId, entry.file);
|
|
253
|
+
}
|
|
254
|
+
const resultPath = path.join(resultsDir, entry.file);
|
|
255
|
+
if (pendingState === "promoted" || existingResultFile(resultPath)) return { file: entry.file, path: resultPath, state: "public" };
|
|
256
|
+
return undefined;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function readResultIndexForSessionRun(resultsDir: string, sessionId: string, runId: string): ResultIndexEntry | undefined {
|
|
260
|
+
try {
|
|
261
|
+
const entry = parseResultIndexEntry(JSON.parse(fs.readFileSync(resultIndexPath(resultsDir, sessionId, runId), "utf-8")));
|
|
262
|
+
if (!entry || entry.sessionId !== sessionId || entry.runId !== runId) return undefined;
|
|
263
|
+
return entry;
|
|
264
|
+
} catch (error) {
|
|
265
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
266
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") console.error(`Ignoring invalid async result index for '${runId}':`, error);
|
|
267
|
+
return undefined;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export function resultPayloadPathForSessionRun(resultsDir: string, sessionId: string, runId: string): string | undefined {
|
|
272
|
+
const entry = readResultIndexForSessionRun(resultsDir, sessionId, runId);
|
|
273
|
+
return (entry ? resultPayloadLocationFromIndex(resultsDir, entry) : undefined)?.path
|
|
274
|
+
?? pendingResultLocationForSessionRun(resultsDir, sessionId, runId)?.path;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export function resultPayloadPathForMissionObserverRun(resultsDir: string, runId: string): string | undefined {
|
|
278
|
+
try {
|
|
279
|
+
const entry = parseResultIndexEntry(JSON.parse(fs.readFileSync(observerIndexPath(resultsDir, MISSION_OBSERVER, runId), "utf-8")));
|
|
280
|
+
if (!entry || entry.runId !== runId) return undefined;
|
|
281
|
+
return resultPayloadLocationFromIndex(resultsDir, entry)?.path;
|
|
282
|
+
} catch (error) {
|
|
283
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Ignoring invalid async result observer index for '${runId}':`, error);
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export function resultPayloadPathForIndexedRun(resultsDir: string, runId: string): string | undefined {
|
|
289
|
+
const root = path.join(resultsDir, RESULT_INDEX_DIR, SESSION_INDEX_DIR);
|
|
290
|
+
let sessions: fs.Dirent[] = [];
|
|
291
|
+
try {
|
|
292
|
+
sessions = fs.readdirSync(root, { withFileTypes: true });
|
|
293
|
+
} catch (error) {
|
|
294
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
295
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") console.error(`Failed to inspect async result session index root '${root}':`, error);
|
|
296
|
+
}
|
|
297
|
+
for (const session of sessions) {
|
|
298
|
+
if (!session.isDirectory()) continue;
|
|
299
|
+
const entryPath = path.join(root, session.name, `${encodeSegment(runId)}.json`);
|
|
300
|
+
try {
|
|
301
|
+
const entry = parseResultIndexEntry(JSON.parse(fs.readFileSync(entryPath, "utf-8")));
|
|
302
|
+
if (!entry || entry.runId !== runId) continue;
|
|
303
|
+
const location = resultPayloadLocationFromIndex(resultsDir, entry);
|
|
304
|
+
if (location) return location.path;
|
|
305
|
+
} catch (error) {
|
|
306
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Ignoring invalid async result index '${entryPath}':`, error);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return pendingResultLocationForIndexedRun(resultsDir, runId)?.path;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function indexedResultFile(resultsDir: string, entry: ResultIndexEntry, includePending = false): string | undefined {
|
|
313
|
+
const location = resultPayloadLocationFromIndex(resultsDir, entry);
|
|
314
|
+
if (!location) return undefined;
|
|
315
|
+
return includePending || location.state === "public" ? location.file : undefined;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function listIndexFiles(dir: string): string[] {
|
|
319
|
+
let files: string[];
|
|
320
|
+
try {
|
|
321
|
+
files = fs.readdirSync(dir, { withFileTypes: true })
|
|
322
|
+
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
|
|
323
|
+
.map((entry) => path.join(dir, entry.name));
|
|
324
|
+
} catch (error) {
|
|
325
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
326
|
+
if (code === "ENOENT" || code === "ENOTDIR") return [];
|
|
327
|
+
throw error;
|
|
328
|
+
}
|
|
329
|
+
return files;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function resultFilesFromIndexDir(resultsDir: string, dir: string, includePending = false): string[] {
|
|
333
|
+
const candidates = new Set<string>();
|
|
334
|
+
for (const entryPath of listIndexFiles(dir)) {
|
|
335
|
+
try {
|
|
336
|
+
const entry = parseResultIndexEntry(JSON.parse(fs.readFileSync(entryPath, "utf-8")));
|
|
337
|
+
if (!entry) {
|
|
338
|
+
fs.rmSync(entryPath, { force: true });
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
const resultFile = indexedResultFile(resultsDir, entry, includePending);
|
|
342
|
+
if (resultFile) candidates.add(resultFile);
|
|
343
|
+
} catch (error) {
|
|
344
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Ignoring invalid async result index '${entryPath}':`, error);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return [...candidates];
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
export function resultFilesForSession(resultsDir: string, sessionId: string): string[] {
|
|
351
|
+
return resultFilesFromIndexDir(resultsDir, sessionIndexDir(resultsDir, sessionId));
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
function pendingResultFilesForSession(resultsDir: string, sessionId: string): string[] {
|
|
355
|
+
const dir = path.join(resultsDir, RESULT_PENDING_DIR, encodeSegment(sessionId));
|
|
356
|
+
let entries: fs.Dirent[];
|
|
357
|
+
try {
|
|
358
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
359
|
+
} catch (error) {
|
|
360
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
361
|
+
throw error;
|
|
362
|
+
}
|
|
363
|
+
const files = new Set<string>();
|
|
364
|
+
for (const entry of entries) {
|
|
365
|
+
if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
|
|
366
|
+
const pendingPath = path.join(dir, entry.name);
|
|
367
|
+
try {
|
|
368
|
+
const data = JSON.parse(fs.readFileSync(pendingPath, "utf-8")) as Record<string, unknown>;
|
|
369
|
+
const runId = nonEmptyString(data.runId) ?? nonEmptyString(data.id);
|
|
370
|
+
if (runId && nonEmptyString(data.sessionId) === sessionId) files.add(resultFileName(runId));
|
|
371
|
+
} catch (error) {
|
|
372
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Ignoring invalid pending async result '${pendingPath}':`, error);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return [...files];
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
export function resultCandidateFilesForSession(resultsDir: string, sessionId: string): string[] {
|
|
379
|
+
return [...new Set([
|
|
380
|
+
...resultFilesFromIndexDir(resultsDir, sessionIndexDir(resultsDir, sessionId), true),
|
|
381
|
+
...pendingResultFilesForSession(resultsDir, sessionId),
|
|
382
|
+
])];
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
export function resultFilesForToolCall(resultsDir: string, toolCallId: string): string[] {
|
|
386
|
+
return resultFilesFromIndexDir(resultsDir, toolCallIndexDir(resultsDir, toolCallId));
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
export function missionObserverResultFiles(resultsDir: string): string[] {
|
|
390
|
+
return resultFilesFromIndexDir(resultsDir, observerIndexDir(resultsDir, MISSION_OBSERVER));
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
export function missionObserverResultCandidateFiles(resultsDir: string): string[] {
|
|
394
|
+
return resultFilesFromIndexDir(resultsDir, observerIndexDir(resultsDir, MISSION_OBSERVER), true);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
export function cleanupResultIndexes(resultsDir: string, now = Date.now(), maxAgeMs = 24 * 60 * 60 * 1000): number {
|
|
398
|
+
const root = path.join(resultsDir, RESULT_INDEX_DIR);
|
|
399
|
+
const cutoff = now - maxAgeMs;
|
|
400
|
+
let removed = 0;
|
|
401
|
+
const visit = (dir: string): void => {
|
|
402
|
+
let entries: fs.Dirent[];
|
|
403
|
+
try {
|
|
404
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
405
|
+
} catch (error) {
|
|
406
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
407
|
+
throw error;
|
|
408
|
+
}
|
|
409
|
+
for (const entry of entries) {
|
|
410
|
+
const fullPath = path.join(dir, entry.name);
|
|
411
|
+
if (entry.isDirectory()) {
|
|
412
|
+
visit(fullPath);
|
|
413
|
+
try { fs.rmdirSync(fullPath); } catch {}
|
|
414
|
+
continue;
|
|
415
|
+
}
|
|
416
|
+
if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
|
|
417
|
+
try {
|
|
418
|
+
const stat = fs.statSync(fullPath);
|
|
419
|
+
const index = parseResultIndexEntry(JSON.parse(fs.readFileSync(fullPath, "utf-8")));
|
|
420
|
+
const resultFile = index ? indexedResultFile(resultsDir, index) : undefined;
|
|
421
|
+
const pendingFile = index ? pendingResultExists(resultsDir, index.sessionId, index.runId) : false;
|
|
422
|
+
if (!index || (!resultFile && !pendingFile && stat.mtimeMs <= cutoff)) {
|
|
423
|
+
fs.rmSync(fullPath, { force: true });
|
|
424
|
+
removed += 1;
|
|
425
|
+
}
|
|
426
|
+
} catch (error) {
|
|
427
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") console.error(`Ignoring invalid async result index '${fullPath}':`, error);
|
|
428
|
+
try {
|
|
429
|
+
fs.rmSync(fullPath, { force: true });
|
|
430
|
+
removed += 1;
|
|
431
|
+
} catch {}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
visit(root);
|
|
436
|
+
return removed;
|
|
437
|
+
}
|