tinker-agent 2.13.0 → 3.0.0

package/src/agent/runtime-hosted-session.ts

@@ -1,3 +1,15 @@
+import { SessionStore } from "../session/session-store";
+import type { SessionLease } from "../session/session-lock";
+import { runtimeIdFactory } from "../ids/runtime-id";
+import { encodeFailure } from "../remote/failures";
+import type { ClientModelCatalog } from "../client/model-catalog";
+import {
+  isSessionOperation,
+  type SessionOperationResult,
+} from "../remote/session-operations";
+import { TuiProjectionStore } from "../tui/tui-projection-store";
+import { ResumeProjectionReader } from "../session/resume-projection";
+import type { RemoteTuiSnapshot } from "../remote/tui-protocol";
 import { randomUUID } from "node:crypto";
 import type { RuntimeSession } from "./runtime-session";
 import type {
@@ -21,20 +33,42 @@ import {
 import { type ManagedSessionRecord, RemoteServiceStore } from "../remote/service-store";
 import { RemoteSyncHub } from "../remote/sync-hub";
 
-export type HostedRuntimeFactory = (input: {
+export type HostedRuntimeFactory = ((input: {
   record: ManagedSessionRecord;
+  lease?: SessionLease;
   sink: EventSink & AssistantTextDeltaSink;
 }) => Promise<{
   runtime: RuntimeSession;
   databasePath: string;
   modelName: string;
-}>;
+  profileName?: string;
+  modelCatalog?: ClientModelCatalog;
+}>) & {
+  profiles?: (workspaceId: string) => Promise<ClientModelCatalog | undefined>;
+  persistDefaultProfile?: (workspaceId: string, profileName: string) => Promise<void>;
+};
+
+export type HostedResources = {
+  load(): Promise<void>;
+  unload(): void;
+  acquireTurn(signal: AbortSignal): Promise<() => void>;
+};
 
 export class HostedSession implements EventSink, AssistantTextDeltaSink {
   readonly name = "remote-view";
   readonly hub: RemoteSyncHub;
   private runtime?: RuntimeSession;
+  private lease?: SessionLease;
+  private suspending?: Promise<void>;
+  private lastAccess = Date.now();
+  private controls = 0;
+  private requests = 0;
+  private closePromise?: Promise<void>;
+  private maintenance?: Promise<unknown>;
+  private modelCatalog?: ClientModelCatalog;
+  private interactionSource?: object;
   private reader?: RemoteHistoryReader;
+  private projection?: TuiProjectionStore;
   private opening?: Promise<void>;
   private readonly queue: OperationReceipt[] = [];
   private active?: {
@@ -49,6 +83,7 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
   private lastOrdinal = 0;
   private streamTimer?: ReturnType<typeof setTimeout>;
   private stopping = false;
+  private connections = 0;
   private unsubscribers: (() => void)[] = [];
   private controlTail: Promise<void> = Promise.resolve();
 
@@ -57,17 +92,80 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
     private readonly store: RemoteServiceStore,
     epoch: string,
     private readonly factory: HostedRuntimeFactory,
+    private readonly resources?: HostedResources,
   ) {
     this.hub = new RemoteSyncHub(epoch, () => this.view());
   }
 
+  assertAvailable(): void {
+    if (this.stopping)
+      throw new RemoteError(409, "SESSION_CLOSING", "Session is closing.");
+  }
+  get connectedClients(): number {
+    return this.connections;
+  }
+  attach(request = false): () => void {
+    this.assertAvailable();
+    this.lastAccess = Date.now();
+    this.connections += 1;
+    if (request) this.requests++;
+    let released = false;
+    return () => {
+      if (!released) {
+        released = true;
+        this.connections -= 1;
+        if (request) this.requests--;
+        this.lastAccess = Date.now();
+      }
+    };
+  }
+  beginDelete(): void {
+    this.assertIdle();
+    if (this.connections > 0)
+      throw new RemoteError(
+        409,
+        "SESSION_CONNECTED",
+        "Cannot delete a session while a client is connected. Disconnect it first.",
+      );
+    this.stopping = true;
+  }
   open(): Promise<void> {
-    return (this.opening ??= this.initialize());
+    this.assertAvailable();
+    this.lastAccess = Date.now();
+    if (this.suspending) return this.suspending.then(() => this.open());
+    return (this.opening ??= this.initialize().catch((error: unknown) => {
+      this.opening = undefined;
+      throw error;
+    }));
   }
   private async initialize(): Promise<void> {
     try {
-      const opened = await this.factory({ record: this.record, sink: this });
+      await this.resources?.load();
+      const opened = await this.factory({
+        record: this.record,
+        sink: this,
+        lease: this.lease,
+      });
       this.runtime = opened.runtime;
+      this.lease = this.runtime.retainSessionLease();
+      if (this.record.residentState?.reasoning)
+        this.runtime.setReasoningEffort(this.record.residentState.reasoning);
+      if (this.record.residentState?.yolo !== undefined)
+        this.runtime.setYoloMode(this.record.residentState.yolo);
+      this.modelCatalog = opened.modelCatalog;
+      this.projection = new TuiProjectionStore({
+        sessionId: this.record.id,
+        workspaceRoot: this.record.workspacePath,
+        modelName: opened.modelName,
+        committedItemLimit: 512,
+        initialSnapshot: ResumeProjectionReader.readDatabase({
+          databasePath: opened.databasePath,
+          workspaceRoot: this.record.workspacePath,
+          sessionId: parseSessionId(this.record.id),
+          modelName: opened.modelName,
+          closedTurnsOnly: true,
+        }),
+      });
       this.reader = new RemoteHistoryReader(
         opened.databasePath,
         parseSessionId(this.record.id),
@@ -77,36 +175,14 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
         ...this.record,
         initialized: true,
         modelName: opened.modelName,
+        profileName: opened.profileName,
         status: "idle",
       };
       this.store.saveSession(this.record);
-      const latest = this.reader.latestTurn();
-      if (latest && latest.status !== "open") {
-        this.activity.status = latest.status as RemoteActivity["status"];
-        this.activity.error = latest.error;
-      }
-      // Canonical terminal status wins over a crash between turn commit and receipt update.
-      for (const receipt of this.store.operations(this.record.id)) {
-        const status = receipt.turnId
-          ? this.reader.turnStatus(receipt.turnId)
-          : undefined;
-        if (receipt.kind === "prompt" && status && status !== "open") {
-          this.store.update({
-            ...receipt,
-            status: status as OperationReceipt["status"],
-            error: status === "interrupted" ? receipt.error : undefined,
-          });
-        }
-      }
-      const last = this.store
-        .operations(this.record.id)
-        .filter((op) => op.kind === "prompt")
-        .at(-1);
-      if (last?.status === "interrupted") {
-        this.activity.status = "interrupted";
-        this.activity.error = last.error;
-      }
+      this.restoreActivity(this.reader);
       this.unsubscribers = [
+        this.runtime.subscribePromptScheduler(() => this.publish()),
+        this.runtime.subscribeProviderRetry(() => this.updateInteraction()),
         this.runtime.subscribeAskUser(() => this.updateInteraction()),
         this.runtime.subscribeBashGuard(() => this.updateInteraction()),
       ];
@@ -118,14 +194,95 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
         await this.runtime
           .dispose({ type: "runner_failed", error: errorMessage(error) })
           .catch(() => undefined);
+      this.reader?.close();
+      this.reader = undefined;
+      this.projection = undefined;
+      this.runtime = undefined;
+      this.resources?.unload();
       this.publish();
       throw error;
     }
   }
 
+  private restoreActivity(reader: RemoteHistoryReader): void {
+    this.activity = { status: "idle", tools: [] };
+    const latest = reader.latestTurn();
+    if (latest && latest.status !== "open") {
+      this.activity.status = latest.status as RemoteActivity["status"];
+      this.activity.error = latest.error;
+    }
+    // Canonical terminal status wins over a crash between turn commit and receipt update.
+    for (const receipt of this.store.operations(this.record.id)) {
+      const status = receipt.turnId ? reader.turnStatus(receipt.turnId) : undefined;
+      // A terminal turn cannot prove that an extended execution chain finished.
+      // Without its final result, a crash may have lost unconsumed follow-ups.
+      if (receipt.hasFollowUps && !receipt.result && receipt.status === "interrupted")
+        continue;
+      if (
+        receipt.kind === "prompt" &&
+        !receipt.shutdownInterrupted &&
+        status &&
+        status !== "open"
+      ) {
+        this.store.update({
+          ...receipt,
+          status: status as OperationReceipt["status"],
+          error: status === "interrupted" ? receipt.error : undefined,
+        });
+      }
+    }
+    const last = this.store
+      .operations(this.record.id)
+      .filter((op) => op.kind === "prompt")
+      .at(-1);
+    if (last?.status === "interrupted") {
+      this.activity.status = "interrupted";
+      this.activity.error = last.error;
+    }
+  }
+
   view(): RemoteView {
     return { ...this.readActivity(), history: this.history() };
   }
+  tuiSnapshot(): RemoteTuiSnapshot {
+    if (!this.projection || !this.runtime) throw new Error("Session is not ready.");
+    const frame = this.hub.snapshot();
+    return {
+      version: 1,
+      cursor: { epoch: frame.epoch, sequence: frame.sequence },
+      activity: this.readActivity(),
+      history: this.projection.getSnapshot(),
+      timeline: this.projection.getLogSnapshot(),
+      promptScheduler: this.runtime.promptScheduler(),
+      profileName: this.record.profileName,
+      modelCatalog: this.modelCatalog,
+      reasoningEffort: this.runtime.reasoningEffort(),
+      supportsImageInput: this.runtime.supportsImageInput(),
+      bashGuard: {
+        ...this.runtime.bashGuard(),
+        ...(this.runtime.bashGuard().pending
+          ? { interactionId: this.activity.interaction?.id }
+          : {}),
+      },
+      askUser: {
+        ...this.runtime.askUser(),
+        ...(this.runtime.askUser().pending
+          ? { interactionId: this.activity.interaction?.id }
+          : {}),
+      },
+      providerRetry:
+        this.runtime.providerRetry().pending && this.activity.interaction
+          ? {
+              pending: {
+                ...this.runtime.providerRetry().pending!,
+                requestId: this.activity.interaction.id,
+              },
+            }
+          : {},
+      skills: this.runtime.skills(),
+      mcp: this.runtime.mcp(),
+    };
+  }
   history(before?: number, limit?: number): RemoteHistoryPage {
     return this.reader?.page(before, limit) ?? { messages: [], hasMore: false };
   }
@@ -135,11 +292,78 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
     return structuredClone({
       ...this.activity,
       session: { ...session, status: this.activity.status },
-      operations: this.store.operations(this.record.id),
+      // Final results are fetched by operation ID, not repeated in every live frame.
+      operations: this.store.operations(this.record.id, false),
     });
   }
 
+  updateModelCatalog(catalog: ClientModelCatalog | undefined): void {
+    this.modelCatalog = catalog;
+    this.publish();
+  }
+
+  assertIdle(): void {
+    if (
+      this.stopping ||
+      this.suspending ||
+      this.maintenance ||
+      this.active ||
+      this.queue.length ||
+      !this.runtime?.canSwitchSession()
+    )
+      throw new RemoteError(
+        409,
+        "SESSION_BUSY",
+        "A turn, context operation or background task is active.",
+      );
+  }
+  get id() {
+    return parseSessionId(this.record.id);
+  }
+  get hasTurns(): boolean {
+    return this.reader?.latestTurn() !== undefined;
+  }
+  exclusive<T>(operation: (runtime: RuntimeSession) => Promise<T>): Promise<T> {
+    this.assertIdle();
+    const pending = Promise.resolve()
+      .then(() => operation(this.runtime!))
+      .finally(() => {
+        if (this.maintenance === pending) this.maintenance = undefined;
+        this.publish();
+      });
+    this.maintenance = pending;
+    return pending;
+  }
+  runMaintenance(
+    receipt: OperationReceipt,
+    operation: (runtime: RuntimeSession) => Promise<SessionOperationResult>,
+  ): void {
+    const running = this.updateReceipt(receipt, { status: "running" });
+    void this.exclusive(operation).then(
+      (sessionResult) => {
+        this.saveResidentState();
+        this.updateReceipt(running, { status: "completed", sessionResult });
+        this.publish();
+      },
+      (error: unknown) => {
+        this.updateReceipt(running, {
+          status: "failed",
+          error: errorMessage(error),
+          failure: encodeFailure(error),
+        });
+        this.publish();
+      },
+    );
+  }
+
   validate(input: RemoteOperationInput): void {
+    if (isSessionOperation(input)) this.assertIdle();
+    if (input.kind === "prompt" && this.maintenance)
+      throw new RemoteError(
+        409,
+        "SESSION_BUSY",
+        "A session maintenance operation is active.",
+      );
     if (this.stopping)
       throw new RemoteError(503, "SERVICE_STOPPING", "The local service is stopping.");
     if (input.kind === "prompt" && this.queue.length >= 8)
@@ -148,12 +372,17 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
         "QUEUE_FULL",
         "This session already has eight queued requests.",
       );
-    if (input.kind === "answer" || input.kind === "confirm") {
+    if (
+      input.kind === "answer" ||
+      input.kind === "confirm" ||
+      input.kind === "provider_retry"
+    ) {
       const pending = this.activity.interaction;
       if (
         !pending ||
         pending.id !== input.interactionId ||
-        pending.kind !== (input.kind === "answer" ? "question" : "confirmation")
+        pending.kind !== (input.kind === "confirm" ? "confirmation" : "question") ||
+        (input.kind === "provider_retry" && !this.runtime?.providerRetry().pending)
       )
         throw new RemoteError(
           409,
@@ -168,6 +397,16 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
       )
         throw new RemoteError(400, "INVALID_ANSWER", "Answer index is out of range.");
     }
+    if (
+      input.kind === "follow_up" &&
+      (this.active?.receipt.requestId !== input.targetRequestId ||
+        this.runtime?.promptScheduler().state !== "running")
+    )
+      throw new RemoteError(
+        409,
+        "STALE_TURN",
+        "The targeted execution is no longer running.",
+      );
     if (input.kind === "stop") {
       const target = this.store.get(input.targetRequestId);
       if (target.sessionId !== this.record.id || target.kind !== "prompt")
@@ -205,7 +444,9 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
     active.completion = this.execute(active);
   }
   private async execute(active: NonNullable<HostedSession["active"]>): Promise<void> {
+    let release: (() => void) | undefined;
     try {
+      release = await this.resources?.acquireTurn(active.controller.signal);
       await this.open();
       if (active.controller.signal.aborted) {
         this.updateReceipt(active.receipt, { status: "cancelled" });
@@ -219,7 +460,13 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
       this.updateReceipt(active.receipt, { status: "running" });
       this.publish();
       const accepted = await this.runtime!.admitTurn({
-        userMessage: { role: "user", content: active.receipt.prompt! },
+        userMessage: {
+          role: "user",
+          content: active.receipt.prompt!,
+          ...(active.receipt.attachments
+            ? { attachments: active.receipt.attachments }
+            : {}),
+        },
         signal: active.controller.signal,
       });
       active.receipt = this.updateReceipt(active.receipt, {
@@ -231,6 +478,7 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
       const result = await accepted.completion;
       this.updateReceipt(active.receipt, {
         status: result.status,
+        result,
         ...(result.status === "failed" ? { error: result.error } : {}),
       });
       this.activity.status = result.status;
@@ -240,6 +488,7 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
       this.updateReceipt(active.receipt, {
         status,
         error: errorMessage(error),
+        failure: encodeFailure(error),
       });
       this.activity.status = status;
       this.activity.error = errorMessage(error);
@@ -253,6 +502,8 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
           tool.status = this.activity.status === "cancelled" ? "cancelled" : "failed";
       }
       this.active = undefined;
+      this.lastAccess = Date.now();
+      release?.();
       this.publish(true);
       this.pump();
     }
@@ -260,38 +511,66 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
 
   control(input: RemoteOperationInput, receipt: OperationReceipt): void {
     // Separate from prompt execution: answering a wait cannot wait for that turn.
-    this.controlTail = this.controlTail.then(async () => {
-      try {
-        await this.open();
-        this.validate(input);
-        if (input.kind === "stop") {
-          if (this.active?.receipt.requestId === input.targetRequestId)
-            this.active.controller.abort();
-          const index = this.queue.findIndex(
-            (op) => op.requestId === input.targetRequestId,
-          );
-          if (index !== -1)
-            this.updateReceipt(this.queue.splice(index, 1)[0], {
-              status: "cancelled",
+    this.controls++;
+    this.controlTail = this.controlTail
+      .then(async () => {
+        try {
+          await this.open();
+          this.validate(input);
+          if (input.kind === "stop") {
+            if (this.active?.receipt.requestId === input.targetRequestId)
+              this.active.controller.abort();
+            const index = this.queue.findIndex(
+              (op) => op.requestId === input.targetRequestId,
+            );
+            if (index !== -1)
+              this.updateReceipt(this.queue.splice(index, 1)[0], {
+                status: "cancelled",
+              });
+          } else if (input.kind === "follow_up") {
+            const followUp = this.runtime!.queueFollowUp({
+              role: "user",
+              content: input.prompt,
             });
-        } else if (input.kind === "answer") {
-          await this.runtime!.resolveAskUser(
-            input.selectedIndex === null
-              ? { outcome: "dismissed" }
-              : { outcome: "selected", selectedIndex: input.selectedIndex },
-          );
-        } else if (input.kind === "confirm") {
-          await this.runtime!.resolveBashConfirmation(input.decision);
+            this.updateReceipt(this.active!.receipt, { hasFollowUps: true });
+            this.updateReceipt(receipt, { status: "completed", followUp });
+            this.publish();
+            return;
+          } else if (
+            input.kind === "provider_retry" ||
+            (input.kind === "answer" && this.runtime!.providerRetry().pending)
+          ) {
+            const retry = this.runtime!.providerRetry().pending!;
+            await this.runtime!.resolveProviderRetry(
+              retry.requestId,
+              input.kind === "provider_retry"
+                ? input.decision
+                : input.selectedIndex === 0
+                  ? "retry"
+                  : "stop",
+            );
+          } else if (input.kind === "answer") {
+            await this.runtime!.resolveAskUser(
+              input.selectedIndex === null
+                ? { outcome: "dismissed" }
+                : { outcome: "selected", selectedIndex: input.selectedIndex },
+            );
+          } else if (input.kind === "confirm") {
+            await this.runtime!.resolveBashConfirmation(input.decision);
+          }
+          this.updateReceipt(receipt, { status: "completed" });
+        } catch (error) {
+          this.updateReceipt(receipt, {
+            status: "failed",
+            error: errorMessage(error),
+          });
         }
-        this.updateReceipt(receipt, { status: "completed" });
-      } catch (error) {
-        this.updateReceipt(receipt, {
-          status: "failed",
-          error: errorMessage(error),
-        });
-      }
-      this.publish();
-    });
+        this.publish();
+      })
+      .finally(() => {
+        this.controls--;
+        this.lastAccess = Date.now();
+      });
   }
 
   private updateReceipt(
@@ -312,14 +591,28 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
   private updateInteraction(): void {
     const question = this.runtime?.askUser().pending;
     const confirmation = this.runtime?.bashGuard().pending;
-    if (question || confirmation) {
-      if (!this.activity.interaction) {
+    const retry = this.runtime?.providerRetry().pending;
+    const source = question ?? confirmation ?? retry;
+    if (source) {
+      if (source !== this.interactionSource) {
+        this.interactionSource = source;
         this.activity.interaction = question
           ? { id: randomUUID(), kind: "question", ...question }
-          : { id: randomUUID(), kind: "confirmation", ...confirmation! };
+          : confirmation
+            ? { id: randomUUID(), kind: "confirmation", ...confirmation }
+            : {
+                id: randomUUID(),
+                kind: "question",
+                question: `Provider request failed. Automatic retries exhausted. ${retry!.failure.error.slice(0, 500)}`,
+                options: [
+                  { description: "Retry again" },
+                  { description: "End this turn" },
+                ],
+              };
       }
       this.activity.status = "waiting_input";
     } else {
+      this.interactionSource = undefined;
       this.activity.interaction = undefined;
       if (this.active) this.activity.status = "running";
     }
@@ -332,6 +625,7 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
 
   async append(event: AgentEvent): Promise<void> {
     if (!this.reader) return;
+    await this.projection?.append(event);
     switch (event.type) {
       case "turn.started":
         this.activity.activeTurnId = event.turnId;
@@ -370,11 +664,12 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
       case "turn.steering.applied":
         break;
       default:
-        return;
+        break;
     }
     this.publish();
   }
   updateAssistantTextDelta(update: AssistantTextDeltaUpdate): void {
+    this.projection?.updateAssistantTextDelta(update);
     const previous = this.activity.streaming;
     this.activity.streaming = {
       iterationId: update.iterationId,
@@ -407,25 +702,154 @@ export class HostedSession implements EventSink, AssistantTextDeltaSink {
     this.hub.publish({ activity: this.readActivity(), messages });
   }
 
-  async close(): Promise<void> {
+  get lastUsed(): number {
+    return this.lastAccess;
+  }
+  get reclaimable(): boolean {
+    return (
+      !!this.runtime &&
+      !this.stopping &&
+      !this.suspending &&
+      !this.connections &&
+      !this.active &&
+      !this.queue.length &&
+      !this.maintenance &&
+      !this.controls &&
+      this.runtime.canSwitchSession()
+    );
+  }
+  get busy(): boolean {
+    return (
+      !!this.active ||
+      !!this.queue.length ||
+      !!this.maintenance ||
+      !!this.controls ||
+      !!this.requests ||
+      (!!this.opening && !this.runtime) ||
+      (!!this.runtime && !this.runtime.canSwitchSession())
+    );
+  }
+  async prepareDormant(homeRoot?: string): Promise<void> {
+    try {
+      await this.recoverDormant(homeRoot);
+    } catch (error) {
+      this.activity = { status: "failed", tools: [], error: errorMessage(error) };
+      throw error;
+    }
+  }
+  private async recoverDormant(homeRoot?: string): Promise<void> {
+    if (!this.record.initialized) return;
+    const store = await SessionStore.openExisting({
+      workspaceRoot: this.record.workspacePath,
+      homeRoot,
+      sessionId: this.id,
+    });
+    this.lease = store.retainLease();
+    try {
+      store.recoverInterruptedState(runtimeIdFactory);
+    } finally {
+      await store.close("session_switch");
+    }
+    const reader = new RemoteHistoryReader(
+      store.databasePath,
+      this.id,
+      this.record.workspacePath,
+    );
+    try {
+      this.restoreActivity(reader);
+    } finally {
+      reader.close();
+    }
+    this.record = { ...this.record, status: this.activity.status };
+    this.store.saveSession(this.record);
+  }
+  suspend(): Promise<void> {
+    if (this.suspending) return this.suspending;
+    if (!this.reclaimable) return Promise.resolve();
+    this.suspending = this.unload().finally(() => {
+      this.suspending = undefined;
+    });
+    return this.suspending;
+  }
+  private saveResidentState(): void {
+    if (!this.runtime) return;
+    const reasoning = this.runtime.reasoningEffort();
+    const guard = this.runtime.bashGuard();
+    this.record = {
+      ...this.record,
+      status: this.activity.status,
+      residentState: {
+        ...(reasoning?.source === "session_override"
+          ? { reasoning: reasoning.effort }
+          : {}),
+        ...(guard.source === "session" ? { yolo: guard.mode === "yolo" } : {}),
+      },
+    };
+    this.store.saveSession(this.record);
+  }
+  private async unload(): Promise<void> {
+    const runtime = this.runtime!;
+    this.saveResidentState();
+    try {
+      await runtime.dispose({ type: "session_switch" });
+    } finally {
+      for (const unsubscribe of this.unsubscribers.splice(0)) unsubscribe();
+      this.reader?.close();
+      this.reader = undefined;
+      this.projection = undefined;
+      this.modelCatalog = undefined;
+      this.interactionSource = undefined;
+      this.activity = {
+        status: this.activity.status,
+        error: this.activity.error,
+        tools: [],
+      };
+      this.runtime = undefined;
+      this.opening = undefined;
+      this.hub.close();
+      this.resources?.unload();
+    }
+  }
+  close(): Promise<void> {
+    return (this.closePromise ??= this.performClose());
+  }
+  private async performClose(): Promise<void> {
     this.stopping = true;
+    this.saveResidentState();
+    const interrupted = this.active?.receipt;
+    this.active?.controller.abort();
     for (const receipt of this.queue.splice(0))
       this.updateReceipt(receipt, {
         status: "interrupted",
-        error: "The service was stopped before this queued request began.",
+        error: "The service stopped before this queued request began.",
       });
+    await this.suspending?.catch(() => undefined);
     if (this.opening) await this.opening.catch(() => undefined);
-    if (this.runtime)
-      await this.runtime.dispose({
-        type: "runner_failed",
-        error: "Remote service shutdown.",
-      });
-    await this.active?.completion;
-    await this.controlTail;
-    for (const unsubscribe of this.unsubscribers) unsubscribe();
-    if (this.streamTimer) clearTimeout(this.streamTimer);
-    this.hub.close();
-    this.reader?.close();
+    try {
+      if (this.runtime)
+        await this.runtime.dispose({
+          type: "runner_failed",
+          error: "Remote service shutdown.",
+        });
+      await this.maintenance?.catch(() => undefined);
+      await this.active?.completion;
+      if (interrupted) {
+        this.updateReceipt(interrupted, {
+          status: "interrupted",
+          shutdownInterrupted: true,
+          error: "Service shutdown interrupted this request; it will not be replayed.",
+        });
+        this.activity.status = "interrupted";
+      }
+      await this.controlTail;
+    } finally {
+      for (const unsubscribe of this.unsubscribers.splice(0)) unsubscribe();
+      if (this.streamTimer) clearTimeout(this.streamTimer);
+      this.hub.close();
+      this.reader?.close();
+      this.resources?.unload();
+      await this.lease?.releaseOwnership();
+    }
   }
   get pendingCount(): number {
     return this.queue.length + (this.active ? 1 : 0);