rea-agents 6.0.0 → 6.2.0

package/dist/hopper/HopperClient.js

@@ -2,17 +2,21 @@ import { randomBytes, randomUUID } from "node:crypto";
 import { access } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import writeFileAtomic from "write-file-atomic";
+import { analysisErrorWithCleanupFailure } from "../application/binary/AnalysisClientCleanup.js";
 import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
 import { HopperCancelledError, HopperProcessError, HopperProtocolError, HopperRemoteError, HopperStartError, HopperTimeoutError, } from "../domain/hopperErrors.js";
 import {} from "../domain/analysisErrorBase.js";
 import { hopperStartupFailure, } from "../domain/hopperStartupFailure.js";
+import { providerRetryAction, sharedProviderFailureStage, } from "../domain/providerOperationHealth.js";
 import { err, ok } from "../domain/result.js";
 import { silentLogger } from "../logger.js";
 import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js";
 import { ProviderStartupDeadline } from "../process/ProviderDeadline.js";
 import { ProviderRunLineage } from "../process/ProviderRunLineage.js";
 import { ProviderProcessSupervisor, } from "../process/ProviderProcess.js";
-import { cleanupHopperSession } from "./HopperCleanup.js";
+import { hopperOperationStage } from "./HopperOperationStage.js";
+import { cleanupHopperSession, } from "./HopperCleanup.js";
 import { parseHopperServerInfo, } from "./HopperSessionValues.js";
 import { connectHopperSocketOnce } from "./HopperSocketConnection.js";
 import { hopperLauncherFailureDiagnostic } from "./HopperProcessDiagnostic.js";
@@ -21,7 +25,6 @@ import { HopperResponseStream } from "./HopperResponseStream.js";
 import { responseResult, } from "./protocol.js";
 const SHUTDOWN_TIMEOUT_MS = 30_000;
 const SESSION_ROOT = process.platform === "darwin" ? "/tmp" : tmpdir();
-/** Safe launcher telemetry; stderr content is intentionally never exposed. */
 /**
  * Owns one authenticated NDJSON-over-Unix-socket bridge session.
  *
@@ -36,15 +39,19 @@ export class HopperClient {
     #responses;
     #logger;
     #socket;
-    #launch;
-    #process;
-    #runtimeRoot;
+    #resources = {
+        launch: undefined,
+        processSupervisor: undefined,
+        runtimeRoot: undefined,
+        shutdownConfirmed: false,
+    };
     #token;
     #lineage = new ProviderRunLineage();
     #nextId = 1;
     #closing = false;
     #launcherExitCode;
     #launcherFailureDiagnostic;
+    #operationFailure;
     #startupController;
     #startPromise;
     #closePromise;
@@ -52,11 +59,31 @@ export class HopperClient {
         this.#responses.push(chunk);
     };
     #onSocketError = () => {
-        this.#failAll(new HopperProcessError(null));
+        const error = new HopperProcessError(null, undefined, undefined, undefined, "unreachable");
+        this.#failAll(error);
+        this.#rememberProcessFailure(error, []);
     };
     #onSocketClose = () => {
-        if (!this.#closing)
-            this.#failAll(new HopperProcessError(null));
+        if (this.#closing)
+            return;
+        const launch = this.#resources.launch;
+        if (launch?.providerLifetime === "launcher-process" &&
+            this.#launcherExitCode !== undefined)
+            return;
+        if (launch?.providerLifetime === "launcher-process") {
+            setImmediate(() => {
+                if (this.#closing || this.#launcherExitCode !== undefined)
+                    return;
+                const exitCode = launch.process.exitCode ?? null;
+                const error = new HopperProcessError(exitCode, this.#launcherFailureDiagnostic, undefined, undefined, exitCode !== null ? "exited" : "unreachable");
+                this.#failAll(error);
+                this.#rememberProcessFailure(error, []);
+            });
+            return;
+        }
+        const error = new HopperProcessError(null, undefined, undefined, undefined, "unreachable");
+        this.#failAll(error);
+        this.#rememberProcessFailure(error, []);
     };
     constructor(options) {
         this.#logger = options.logger ?? silentLogger;
@@ -91,10 +118,60 @@ export class HopperClient {
     requestActivity() {
         return this.#requests.activity();
     }
+    /**
+     * Latest operational health for this owned bridge.
+     * A recorded exit remains visible after request activity returns to idle.
+     */
+    operationHealth() {
+        const failure = this.#operationFailure;
+        if (failure !== undefined) {
+            const requests = failure.requests.map((request) => ({
+                ...request,
+                stage: failure.startupFailure
+                    ? "launch"
+                    : hopperOperationStage(request.operation),
+            }));
+            return {
+                state: failure.state,
+                stage: requests.length === 0
+                    ? failure.startupFailure
+                        ? "launch"
+                        : "connection"
+                    : sharedProviderFailureStage(requests),
+                retryAction: providerRetryAction(failure.state, failure.startupFailure),
+                exitCode: failure.exitCode,
+                requests,
+            };
+        }
+        const activity = this.requestActivity();
+        if (activity !== null) {
+            const stage = hopperOperationStage(activity.operation);
+            return {
+                state: "busy",
+                stage,
+                retryAction: "wait",
+                exitCode: null,
+                requests: [
+                    {
+                        requestId: activity.requestId,
+                        operation: activity.operation,
+                        stage,
+                    },
+                ],
+            };
+        }
+        return {
+            state: "idle",
+            stage: null,
+            retryAction: null,
+            exitCode: null,
+            requests: [],
+        };
+    }
     /** Launch the bridge once and complete its authenticated health handshake. */
     start(signal) {
         if (this.#closePromise !== undefined) {
-            return this.#closePromise.then(() => this.start(signal));
+            return this.#closePromise.then((closed) => closed.ok ? this.start(signal) : closed);
         }
         if (this.#startPromise !== undefined)
             return this.#startPromise;
@@ -120,8 +197,10 @@ export class HopperClient {
         };
         void started.then((result) => {
             release();
-            if (!result.ok)
+            if (!result.ok) {
+                this.#rememberStartupFailure(result.error);
                 reset();
+            }
         }, (cause) => {
             this.#logger.debug({ error: cause instanceof Error ? cause.message : String(cause) }, "Hopper startup promise rejected during bookkeeping");
             release();
@@ -132,9 +211,13 @@ export class HopperClient {
     async #start(signal) {
         if (isAborted(signal))
             return err(new HopperCancelledError());
-        if (this.#socket !== undefined || this.#runtimeRoot !== undefined) {
+        this.#operationFailure = undefined;
+        if (this.#socket !== undefined ||
+            this.#resources.launch !== undefined ||
+            this.#resources.runtimeRoot !== undefined) {
             return err(new HopperProtocolError("Hopper client is already started"));
         }
+        this.#resources.shutdownConfirmed = false;
         const deadline = new ProviderStartupDeadline(this.#options.startupTimeoutMs, signal);
         try {
             return await this.#startWithin(deadline);
@@ -145,7 +228,7 @@ export class HopperClient {
     }
     async #startWithin(deadline) {
         try {
-            this.#runtimeRoot = await PrivateRuntimeRoot.create({
+            this.#resources.runtimeRoot = await PrivateRuntimeRoot.create({
                 parent: SESSION_ROOT,
                 prefix: "rea-",
             });
@@ -154,10 +237,9 @@ export class HopperClient {
             return err(new HopperStartError({ cause }));
         }
         if (deadline.signal.aborted) {
-            await this.#cleanup();
-            return err(startupInterruption(deadline));
+            return this.#startupFailure(startupInterruption(deadline));
         }
-        const socketPath = join(this.#runtimeRoot.path, "bridge.sock");
+        const socketPath = join(this.#resources.runtimeRoot.path, "bridge.sock");
         this.#token = randomBytes(32).toString("hex");
         this.#lineage.reset();
         this.#launcherExitCode = undefined;
@@ -165,47 +247,76 @@ export class HopperClient {
         const runId = this.#options.runId ?? randomUUID();
         const launched = await this.#options.launcher
             .launch({
-            directory: this.#runtimeRoot.path,
+            directory: this.#resources.runtimeRoot.path,
             socketPath,
             token: this.#token,
             runId,
         }, { signal: deadline.signal })
             .catch((cause) => err(new HopperStartError({ cause })));
         if (!launched.ok) {
-            await this.#cleanup();
-            return deadline.signal.aborted
-                ? err(startupInterruption(deadline))
-                : launched;
+            return this.#startupFailure(deadline.signal.aborted
+                ? startupInterruption(deadline)
+                : launched.error);
         }
-        this.#launch = launched.value;
+        this.#resources.launch = launched.value;
         this.#attachLauncher(launched.value);
+        const ownership = launched.value.ownership;
+        if (ownership !== undefined) {
+            try {
+                await writeFileAtomic(join(this.#resources.runtimeRoot.path, "ownership.json"), `${JSON.stringify({
+                    run_id: runId,
+                    pid: ownership.leaderPid,
+                    process_group_id: ownership.processGroupId,
+                    parent_pid: process.pid,
+                    launcher: launched.value.launcherCommand ?? null,
+                    created_at: new Date().toISOString(),
+                })}\n`, { encoding: "utf8", mode: 0o600 });
+            }
+            catch (cause) {
+                return this.#startupFailure(new HopperStartError({ cause }));
+            }
+        }
         const connected = await this.#connect(socketPath, deadline);
         if (!connected.ok) {
-            await this.#cleanup();
-            return connected;
+            return this.#startupFailure(connected.error);
         }
         const remainingMs = deadline.remainingMs();
         if (remainingMs <= 0) {
-            await this.#cleanup();
-            return err(new HopperTimeoutError(this.#options.startupTimeoutMs));
+            return this.#startupFailure(new HopperTimeoutError(this.#options.startupTimeoutMs));
         }
         const health = await this.#request("health", {}, {
             timeoutMs: remainingMs,
             signal: deadline.signal,
         });
         if (!health.ok) {
-            await this.#cleanup();
-            return deadline.signal.aborted
-                ? err(startupInterruption(deadline))
-                : health;
+            return this.#startupFailure(deadline.signal.aborted ? startupInterruption(deadline) : health.error);
         }
         const parsed = parseHopperServerInfo(health.value, runId);
-        if (parsed.ok)
-            await this.#lineage.observe(this.#launch);
-        else
-            await this.#cleanup();
+        if (!parsed.ok)
+            return this.#startupFailure(parsed.error);
+        await this.#lineage.observe(this.#resources.launch);
         return parsed;
     }
+    async #startupFailure(primary) {
+        this.#rememberStartupFailure(primary);
+        const closed = await this.#cleanup();
+        return err(closed.ok
+            ? primary
+            : analysisErrorWithCleanupFailure(primary, closed.error, "open_binary"));
+    }
+    #rememberStartupFailure(failure) {
+        if (failure instanceof HopperProcessError)
+            this.#rememberProcessFailure(failure, []);
+        else if (this.#operationFailure === undefined &&
+            (failure instanceof HopperTimeoutError ||
+                failure instanceof HopperStartError))
+            this.#operationFailure = {
+                state: failure instanceof HopperTimeoutError ? "not_started" : "unknown",
+                exitCode: null,
+                startupFailure: true,
+                requests: [],
+            };
+    }
     /** Invoke one declared operation, returning timeout and cancellation as values. */
     async callTool(name, arguments_ = {}, options = {}) {
         const started = await this.start(options.signal);
@@ -218,12 +329,8 @@ export class HopperClient {
             ? err(new AnalysisCapabilityUnavailableError("hopper", name, result.error.safeMessage))
             : result;
     }
-    /** Stop the bridge, settle outstanding requests, and remove session artifacts. */
-    close() {
-        return this.closeWithOutcome().then(() => undefined);
-    }
     /** Stop the bridge and report whether every owned resource was verified clean. */
-    closeWithOutcome(options = {}) {
+    close(options = {}) {
         const starting = this.#startPromise;
         const controller = this.#startupController;
         controller?.abort();
@@ -251,9 +358,7 @@ export class HopperClient {
         try {
             return await cleanupHopperSession({
                 socket: this.#socket,
-                launch: this.#launch,
-                processSupervisor: this.#process,
-                runtimeRoot: this.#runtimeRoot,
+                resources: this.#resources,
                 activeRequest: this.#requests.activity(),
                 progress: options.progress,
                 logger: this.#logger,
@@ -269,26 +374,34 @@ export class HopperClient {
             });
         }
         finally {
-            this.#socket = undefined;
-            this.#process = undefined;
-            this.#launch = undefined;
-            this.#runtimeRoot = undefined;
-            this.#token = undefined;
-            this.#responses.reset();
-            this.#launcherExitCode = undefined;
-            this.#launcherFailureDiagnostic = undefined;
+            if (this.#socket?.destroyed === true)
+                this.#socket = undefined;
+            if (this.#socket === undefined)
+                this.#token = undefined;
+            if (this.#socket === undefined)
+                this.#responses.reset();
+            if (this.#resources.processSupervisor === undefined) {
+                this.#launcherExitCode = undefined;
+                this.#launcherFailureDiagnostic = undefined;
+            }
             this.#closing = false;
         }
     }
     async #connect(socketPath, deadline) {
         while (deadline.remainingMs() > 0) {
             if (deadline.signal.aborted)
-                return err(startupInterruption(deadline));
+                return err(await this.#startupInterruption(deadline));
             if (this.#closing)
                 return err(new HopperProcessError(null));
             if (this.#launcherExitCode !== undefined &&
-                hopperStartupFailure(this.#launcherExitCode) !== undefined)
-                return err(new HopperProcessError(this.#launcherExitCode, this.#launcherFailureDiagnostic));
+                this.#resources.launch !== undefined &&
+                (this.#resources.launch.providerLifetime === "launcher-process" ||
+                    this.#launcherExitCode !== 0)) {
+                const failure = await this.#launcherStartupFailure(this.#resources.launch.providerLifetime === "launcher-process", this.#launcherExitCode, deadline);
+                return deadline.interruption === "cancelled"
+                    ? err(await this.#startupInterruption(deadline))
+                    : err(failure);
+            }
             try {
                 await access(socketPath);
             }
@@ -296,7 +409,7 @@ export class HopperClient {
                 // best-effort cleanup: socket polling; absence means keep waiting.
                 void cause;
                 if ((await deadline.wait(50)) === "aborted")
-                    return err(startupInterruption(deadline));
+                    return err(await this.#startupInterruption(deadline));
                 continue;
             }
             const attempt = await connectHopperSocketOnce(socketPath, deadline.signal);
@@ -306,24 +419,64 @@ export class HopperClient {
                 return ok(undefined);
             }
             if ((await deadline.wait(50)) === "aborted")
-                return err(startupInterruption(deadline));
+                return err(await this.#startupInterruption(deadline));
         }
-        return err(new HopperTimeoutError(this.#options.startupTimeoutMs));
+        return err(await this.#startupInterruption(deadline));
+    }
+    async #startupInterruption(deadline) {
+        if (deadline.interruption === "cancelled")
+            return new HopperCancelledError();
+        return new HopperTimeoutError(deadline.timeoutMs, undefined, undefined, "not_started", "launch", this.#launcherOutcome((await this.#resources.processSupervisor?.waitForOutputClose(0)) ??
+            false));
+    }
+    #launcherOutcome(outputClosed) {
+        const snapshot = this.#resources.processSupervisor?.snapshot();
+        const token = this.#token;
+        if (snapshot === undefined)
+            return undefined;
+        const redact = (text) => token === undefined
+            ? text
+            : text.replaceAll(token, "[redacted transport credential]");
+        return {
+            exit_code: snapshot.exitCode ?? null,
+            signal: snapshot.signal ?? null,
+            stdout: { ...snapshot.stdout, text: redact(snapshot.stdout.text) },
+            stderr: { ...snapshot.stderr, text: redact(snapshot.stderr.text) },
+            output_closed: outputClosed,
+            diagnostic_truncated: snapshot.diagnosticTruncated === true,
+        };
+    }
+    async #launcherStartupFailure(ownsProviderLifetime, exitCode, deadline) {
+        const process = this.#resources.processSupervisor;
+        // Descendants may inherit the helper's pipes. Bound drainage independently
+        // from readiness, and report whether producer output actually closed.
+        const outputClosed = (await process?.waitForOutputClose(Math.min(1_000, deadline.remainingMs()))) ?? false;
+        const launcherFailure = this.#launcherOutcome(outputClosed);
+        if (ownsProviderLifetime)
+            return new HopperProcessError(exitCode, this.#launcherFailureDiagnostic, undefined, undefined, "exited", "launch", launcherFailure);
+        return new HopperStartError({
+            userMessage: "Hopper's launcher helper failed before bridge readiness. Review details.launcher and complete Hopper setup before retrying.",
+            ...(launcherFailure === undefined ? {} : { launcherFailure }),
+        });
     }
     async #request(method, params, options) {
         const id = this.#nextId++;
         const socket = this.#socket;
         const token = this.#token;
         if (socket === undefined || socket.destroyed || token === undefined) {
-            return err(new HopperProcessError(null, undefined, method, id));
+            return this.#unavailableRequest(new HopperProcessError(null, undefined, method, id, "unreachable", hopperOperationStage(method)));
         }
         if (options.signal?.aborted === true)
             return err(new HopperCancelledError());
         const startedAt = performance.now();
         const result = await this.#requests.run(id, method, params, {
             ...(options.signal === undefined ? {} : { signal: options.signal }),
+            ...(options.timeoutMs === undefined
+                ? {}
+                : { timeoutMs: options.timeoutMs }),
             ...(options.progress !== undefined ? { progress: options.progress } : {}),
         });
+        this.#retainRequestHealth(result, id, method);
         this.#logger[result.ok ? "debug" : "warn"]({
             method,
             durationMs: Math.round((performance.now() - startedAt) * 100) / 100,
@@ -370,7 +523,7 @@ export class HopperClient {
         socket.off("close", this.#onSocketClose);
     }
     #attachLauncher(launch) {
-        this.#process = new ProviderProcessSupervisor(launch, {
+        this.#resources.processSupervisor = new ProviderProcessSupervisor(launch, {
             onDiagnostic: (event) => this.#onLauncherDiagnostic(launch, event),
         });
     }
@@ -394,8 +547,10 @@ export class HopperClient {
             type: "launcher-exit",
             code: event.code,
         });
-        if (launch.ownsProcessLifetime && !this.#closing)
-            this.#failAll(new HopperProcessError(event.code, this.#launcherFailureDiagnostic));
+        if (launch.providerLifetime === "launcher-process" && !this.#closing) {
+            this.#rememberExit(event.code);
+            this.#failAll(new HopperProcessError(event.code, this.#launcherFailureDiagnostic, undefined, undefined, "exited"));
+        }
     }
     #abortProtocol(message, cause) {
         this.#failAll(new HopperProtocolError(message, { cause }));
@@ -404,7 +559,81 @@ export class HopperClient {
     #failAll(error) {
         this.#requests.failAll(error);
     }
+    #unavailableRequest(error) {
+        this.#rememberProcessFailure(error, error.requestId === undefined || error.operation === undefined
+            ? []
+            : [{ requestId: error.requestId, operation: error.operation }]);
+        return err(error);
+    }
+    #retainRequestHealth(result, requestId, operation) {
+        if (this.#closing)
+            return;
+        if (result.ok)
+            return;
+        if (!(result.error instanceof HopperProcessError))
+            return;
+        this.#rememberProcessFailure(result.error, [
+            {
+                requestId: result.error.requestId ?? requestId,
+                operation: result.error.operation ?? operation,
+            },
+        ]);
+    }
+    #rememberExit(code) {
+        if (this.#closing)
+            return;
+        const previous = this.#operationFailure;
+        this.#operationFailure = {
+            state: "exited",
+            exitCode: code ?? previous?.exitCode ?? null,
+            startupFailure: hopperStartupFailure(code) !== undefined ||
+                previous?.startupFailure === true,
+            requests: previous?.requests ?? [],
+        };
+    }
+    #rememberProcessFailure(error, requests) {
+        if (this.#closing)
+            return;
+        const previous = this.#operationFailure;
+        const state = processFailureState(error);
+        const startupFailure = error.stage === "launch" ||
+            error.failureCode !== undefined ||
+            previous?.startupFailure === true;
+        const merged = mergeFailedRequests(previous?.requests ?? [], requests);
+        if (previous !== undefined &&
+            PROCESS_FAILURE_RANK[previous.state] > PROCESS_FAILURE_RANK[state]) {
+            this.#operationFailure = {
+                ...previous,
+                startupFailure,
+                requests: merged,
+            };
+            return;
+        }
+        this.#operationFailure = {
+            state,
+            exitCode: state === "exited"
+                ? (error.exitCode ?? previous?.exitCode ?? null)
+                : null,
+            startupFailure,
+            requests: merged,
+        };
+    }
 }
+const PROCESS_FAILURE_RANK = {
+    not_started: 0,
+    unknown: 0,
+    unreachable: 1,
+    exited: 2,
+};
+const processFailureState = (error) => error.providerState === "exited" || error.exitCode !== null
+    ? "exited"
+    : error.providerState;
+const mergeFailedRequests = (current, added) => {
+    const byId = new Map();
+    for (const request of [...current, ...added])
+        byId.set(request.requestId, request);
+    return [...byId.values()].sort((left, right) => left.requestId - right.requestId);
+};
 const isAborted = (signal) => signal?.aborted === true;
 const startupInterruption = (deadline) => deadline.interruption === "cancelled"
     ? new HopperCancelledError()