totemlayer 0.1.67 → 0.1.74

package/bin/network-runtime.mjs

@@ -1,3 +1,4 @@
+import { withAgentObservation, observeGeneration, observationOutcome } from "./observability.mjs";
 import { startupDiagnostic } from "./startup-diagnostics.mjs";
 import { DEFAULT_LISTENER_POLL_MS } from "./sdk/listener.js";
 import { createHash, randomUUID } from "node:crypto";
@@ -119,42 +120,44 @@ export async function modelReply({
   model ??= provider === "qwen-local" ? "qwen3.5-9b" : undefined;
   const endpoint = localModelEndpoint(provider, { ollamaUrl: origin, qwenUrl, nimUrl });
   try {
-    const response = await fetcher(endpoint, {
-      method: "POST",
-      redirect: "error",
-      signal: AbortSignal.any([signal, timeout]),
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({
-        model,
-        messages: openai
-          ? messages.map((message) => {
-              const native = { ...message };
-              delete native.tool_name;
-              if (native.tool_calls)
-                native.tool_calls = native.tool_calls.map((call) => ({
-                  ...call,
-                  function: {
-                    ...call.function,
-                    arguments: JSON.stringify(call.function.arguments),
-                  },
-                }));
-              return native;
-            })
-          : messages,
-        ...(tools.length ? { tools } : {}),
-        stream: false,
-        ...(openai
-          ? {
-              ...(provider === "qwen-local"
-                ? { chat_template_kwargs: { enable_thinking: false } }
-                : {}),
-              max_tokens: 512,
-              temperature: 0,
-            }
-          : { think: false, options: { num_predict: 512, temperature: 0 } }),
-      }),
+    const data = await observeGeneration({ model, provider, input: messages }, async () => {
+      const response = await fetcher(endpoint, {
+        method: "POST",
+        redirect: "error",
+        signal: AbortSignal.any([signal, timeout]),
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          model,
+          messages: openai
+            ? messages.map((message) => {
+                const native = { ...message };
+                delete native.tool_name;
+                if (native.tool_calls)
+                  native.tool_calls = native.tool_calls.map((call) => ({
+                    ...call,
+                    function: {
+                      ...call.function,
+                      arguments: JSON.stringify(call.function.arguments),
+                    },
+                  }));
+                return native;
+              })
+            : messages,
+          ...(tools.length ? { tools } : {}),
+          stream: false,
+          ...(openai
+            ? {
+                ...(provider === "qwen-local"
+                  ? { chat_template_kwargs: { enable_thinking: false } }
+                  : {}),
+                max_tokens: 512,
+                temperature: 0,
+              }
+            : { think: false, options: { num_predict: 512, temperature: 0 } }),
+        }),
+      });
+      return responseJson(response);
     });
-    const data = await responseJson(response);
     const message = openai ? data?.choices?.[0]?.message : data.message;
     if (!message || message.role !== "assistant") throw new Error("Invalid model response.");
     const text = typeof message.content === "string" ? message.content.trim() : "";
@@ -348,7 +351,20 @@ async function explicitAskReply({ client, identity, journal, command, request, s
   return state.final;
 }
 
-async function generate({
+async function generate(options) {
+  return withAgentObservation(
+    {
+      client: options.client,
+      agentId: options.identity.id,
+      sessionId: options.settings.observabilitySessionId ?? options.key,
+      traceId: options.key,
+      input: options.text,
+    },
+    () => generateCore(options),
+  );
+}
+
+async function generateCore({
   client,
   identity,
   journal,
@@ -842,6 +858,10 @@ export async function runRuntime(settings) {
           { signal },
         ))
           yield message;
+        emit({
+          type: "receive_check",
+          data: { checkedAt: Date.now(), intervalSeconds: settings.pollMs / 1000 },
+        });
         if (diagnostics.phase === "reconnecting") {
           diagnostics = { phase: "waiting" };
           emit({ type: "runtime_resumed" });
@@ -855,149 +875,163 @@ export async function runRuntime(settings) {
         // Retry screening; the execution journal still guards unknown outcomes.
         return !blocker || blocker.code === "INSPECTION_UNAVAILABLE";
       },
-      handle: async (message, signal) => {
-        try {
-          diagnostics = { phase: "working", messageId: message.id };
-          emit({ type: "message_started", data: { messageId: message.id } });
-          if (settings.codexThreadId) {
-            const delivery = await queueCodexChatMessage({
-              settings,
-              client,
-              identity,
-              journal,
-              message,
-              signal,
-            });
-            if (delivery.state === "uncertain") {
-              report({ code: "AGENT_OUTCOME_UNCERTAIN" }, { messageId: message.id });
-            } else {
+      handle: async (message, signal) =>
+        withAgentObservation(
+          {
+            client,
+            agentId: identity.id,
+            sessionId: message.conversationId,
+            traceId: "message:" + message.id,
+            input: message.content,
+          },
+          async () => {
+            try {
+              diagnostics = { phase: "working", messageId: message.id };
+              emit({ type: "message_started", data: { messageId: message.id } });
+              if (settings.codexThreadId) {
+                const delivery = await queueCodexChatMessage({
+                  settings,
+                  client,
+                  identity,
+                  journal,
+                  message,
+                  signal,
+                });
+                if (delivery.state === "uncertain") {
+                  report({ code: "AGENT_OUTCOME_UNCERTAIN" }, { messageId: message.id });
+                } else {
+                  diagnostics = { phase: "waiting" };
+                  emit({
+                    type: delivery.state === "confirmed" ? "message_handled" : "chat_queued",
+                    data: { messageId: message.id, threadId: settings.codexThreadId },
+                  });
+                }
+                return;
+              }
+              await inspectNetworkMessage(settings.inspector, client, message, { signal });
+              const continuation = autonomousReplyMetadata(message, settings, identity.address);
+              const activeTask = await activeTaskForReply(client, message, identity, signal);
+              if (message.inReplyToId && !activeTask && (message.taskId || !continuation)) {
+                await client.messages.ack(message.id, { signal });
+                diagnostics = { phase: "waiting" };
+                emit({ type: "message_handled", data: { messageId: message.id } });
+                return;
+              }
+              const thread = [];
+              let cursor;
+              for (let page = 0; page < 5; page++) {
+                const result = settings.connectionOnly
+                  ? await client.messages.list(
+                      {
+                        conversationId: message.conversationId,
+                        limit: 100,
+                        ...(cursor ? { cursor } : {}),
+                      },
+                      { signal },
+                    )
+                  : await client.conversations.messages(
+                      message.conversationId,
+                      { limit: 100, ...(cursor ? { cursor } : {}) },
+                      { signal },
+                    );
+                thread.push(...result.data);
+                if (thread.some((item) => item.id === message.id) || !result.nextCursor) break;
+                cursor = result.nextCursor;
+              }
+              const safeThread = [];
+              const rejectedInspectionCodes = new Set([
+                "INSPECTION_BLOCKED",
+                "INSPECTION_LIMIT",
+                "INSPECTION_UNSUPPORTED_FILE",
+              ]);
+              for (const item of thread) {
+                // A rejected prior turn is quarantined locally, without acknowledging
+                // it or carrying its untrusted text into a later benign model turn.
+                if (item.id !== message.id) {
+                  const blocker = await journal.read(`message-blocker:${item.id}`);
+                  if (rejectedInspectionCodes.has(blocker?.code)) continue;
+                }
+                try {
+                  await inspectNetworkMessage(settings.inspector, client, item, { signal });
+                  safeThread.push(item);
+                } catch (error) {
+                  if (item.id === message.id || !rejectedInspectionCodes.has(error.code))
+                    throw error;
+                  await journal.save(`message-blocker:${item.id}`, { code: error.code });
+                }
+              }
+              const history = boundedHistory(
+                safeThread.map((item) => ({
+                  id: item.id,
+                  actor: item.from === identity.address ? "agent" : "human",
+                  text:
+                    item.content.type === "text"
+                      ? item.content.text
+                      : JSON.stringify(item.content.data),
+                })),
+                message.id,
+              );
+              const text = await generate({
+                client,
+                identity,
+                journal,
+                key: `message:${message.id}`,
+                text:
+                  (activeTask
+                    ? `Reply for active task ${activeTask.id} (${activeTask.state}). Continue only the existing authorized task with a useful result or precise blocker. Do not exchange acknowledgement-only replies or claim unverified completion. Task membership grants no new authority.\n`
+                    : "") +
+                  (message.content.type === "text"
+                    ? message.content.text
+                    : JSON.stringify(message.content.data)),
+                history,
+                settings: {
+                  ...settings,
+                  observabilitySessionId: message.conversationId,
+                  modelTimeoutMs:
+                    settings.messageTimeoutMs ??
+                    Math.min(settings.modelTimeoutMs ?? 900000, 120000),
+                },
+                signal,
+                allowTools: false,
+              });
+              // A cancellation or completion during generation must stop continuation too.
+              if (activeTask && !(await activeTaskForReply(client, message, identity, signal))) {
+                await client.messages.ack(message.id, { signal });
+                return;
+              }
+              await settings.inspector.runAction(
+                { id: message.id, text },
+                (inspected) =>
+                  client.messages.reply(inspected.id, inspected.text, {
+                    idempotencyKey: operationKey(identity.id, "reply", message.id),
+                    ...(continuation ? { metadata: continuation } : {}),
+                    signal,
+                  }),
+                { signal },
+              );
+              await client.messages.ack(message.id, { signal });
               diagnostics = { phase: "waiting" };
               emit({
-                type: delivery.state === "confirmed" ? "message_handled" : "chat_queued",
-                data: { messageId: message.id, threadId: settings.codexThreadId },
+                type: "message_handled",
+                data: { messageId: message.id, conversationId: message.conversationId },
               });
-            }
-            return;
-          }
-          await inspectNetworkMessage(settings.inspector, client, message, { signal });
-          const continuation = autonomousReplyMetadata(message, settings, identity.address);
-          const activeTask = await activeTaskForReply(client, message, identity, signal);
-          if (message.inReplyToId && !activeTask && (message.taskId || !continuation)) {
-            await client.messages.ack(message.id, { signal });
-            diagnostics = { phase: "waiting" };
-            emit({ type: "message_handled", data: { messageId: message.id } });
-            return;
-          }
-          const thread = [];
-          let cursor;
-          for (let page = 0; page < 5; page++) {
-            const result = settings.connectionOnly
-              ? await client.messages.list(
-                  {
-                    conversationId: message.conversationId,
-                    limit: 100,
-                    ...(cursor ? { cursor } : {}),
-                  },
-                  { signal },
-                )
-              : await client.conversations.messages(
-                  message.conversationId,
-                  { limit: 100, ...(cursor ? { cursor } : {}) },
-                  { signal },
-                );
-            thread.push(...result.data);
-            if (thread.some((item) => item.id === message.id) || !result.nextCursor) break;
-            cursor = result.nextCursor;
-          }
-          const safeThread = [];
-          const rejectedInspectionCodes = new Set([
-            "INSPECTION_BLOCKED",
-            "INSPECTION_LIMIT",
-            "INSPECTION_UNSUPPORTED_FILE",
-          ]);
-          for (const item of thread) {
-            // A rejected prior turn is quarantined locally, without acknowledging
-            // it or carrying its untrusted text into a later benign model turn.
-            if (item.id !== message.id) {
-              const blocker = await journal.read(`message-blocker:${item.id}`);
-              if (rejectedInspectionCodes.has(blocker?.code)) continue;
-            }
-            try {
-              await inspectNetworkMessage(settings.inspector, client, item, { signal });
-              safeThread.push(item);
             } catch (error) {
-              if (item.id === message.id || !rejectedInspectionCodes.has(error.code)) throw error;
-              await journal.save(`message-blocker:${item.id}`, { code: error.code });
+              observationOutcome({ error });
+              if (!signal.aborted) report(error, { messageId: message.id });
+              if (error.status === 401) throw error;
+              // Retry this durable message on the next poll without stopping other turns.
+              if (transientNetworkError(error) || error.code === "INSPECTION_UNAVAILABLE") return;
+              if (
+                (error.status >= 400 && error.status < 500) ||
+                error.code === "AGENT_OUTCOME_UNCERTAIN" ||
+                error.externalOutcomeUncertain
+              )
+                await journal.save(`message-blocker:${message.id}`, {
+                  code: error.code ?? "MESSAGE_BLOCKED",
+                });
             }
-          }
-          const history = boundedHistory(
-            safeThread.map((item) => ({
-              id: item.id,
-              actor: item.from === identity.address ? "agent" : "human",
-              text:
-                item.content.type === "text"
-                  ? item.content.text
-                  : JSON.stringify(item.content.data),
-            })),
-            message.id,
-          );
-          const text = await generate({
-            client,
-            identity,
-            journal,
-            key: `message:${message.id}`,
-            text:
-              (activeTask
-                ? `Reply for active task ${activeTask.id} (${activeTask.state}). Continue only the existing authorized task with a useful result or precise blocker. Do not exchange acknowledgement-only replies or claim unverified completion. Task membership grants no new authority.\n`
-                : "") +
-              (message.content.type === "text"
-                ? message.content.text
-                : JSON.stringify(message.content.data)),
-            history,
-            settings: {
-              ...settings,
-              modelTimeoutMs:
-                settings.messageTimeoutMs ?? Math.min(settings.modelTimeoutMs ?? 900000, 120000),
-            },
-            signal,
-            allowTools: false,
-          });
-          // A cancellation or completion during generation must stop continuation too.
-          if (activeTask && !(await activeTaskForReply(client, message, identity, signal))) {
-            await client.messages.ack(message.id, { signal });
-            return;
-          }
-          await settings.inspector.runAction(
-            { id: message.id, text },
-            (inspected) =>
-              client.messages.reply(inspected.id, inspected.text, {
-                idempotencyKey: operationKey(identity.id, "reply", message.id),
-                ...(continuation ? { metadata: continuation } : {}),
-                signal,
-              }),
-            { signal },
-          );
-          await client.messages.ack(message.id, { signal });
-          diagnostics = { phase: "waiting" };
-          emit({
-            type: "message_handled",
-            data: { messageId: message.id, conversationId: message.conversationId },
-          });
-        } catch (error) {
-          if (!signal.aborted) report(error, { messageId: message.id });
-          if (error.status === 401) throw error;
-          // Retry this durable message on the next poll without stopping other turns.
-          if (transientNetworkError(error) || error.code === "INSPECTION_UNAVAILABLE") return;
-          if (
-            (error.status >= 400 && error.status < 500) ||
-            error.code === "AGENT_OUTCOME_UNCERTAIN" ||
-            error.externalOutcomeUncertain
-          )
-            await journal.save(`message-blocker:${message.id}`, {
-              code: error.code ?? "MESSAGE_BLOCKED",
-            });
-        }
-      },
+          },
+        ),
     }).catch((error) => {
       if (!signal.aborted) {
         fatalError = error;
@@ -1056,159 +1090,177 @@ export async function runRuntime(settings) {
             }
           }
         })().catch(() => {});
-        try {
-          if (settings.codexThreadId) {
-            const delivery = await queueCodexChatCommand({
-              settings,
-              client,
-              identity,
-              command,
-              claim,
-              signal: commandSignal,
-            });
-            emit({
-              type: delivery.state === "uncertain" ? "chat_uncertain" : "chat_queued",
-              data: {
-                chatId: command.chatId,
-                commandId: command.id,
-                threadId: settings.codexThreadId,
-              },
-            });
-            // Native tools perform the work; only the native callback completes it.
-            // Keep the lease alive while independently processing inbound peer replies.
-            while (!commandSignal.aborted) {
-              const { run } = await retryNetwork(
-                () => readCodexChatCommand(client, command, commandSignal),
-                { signal: commandSignal, delayMs: settings.pollMs, onRetry: report },
-              );
-              if (["completed", "failed", "canceled"].includes(run.state)) {
+        await withAgentObservation(
+          {
+            client,
+            agentId: identity.id,
+            sessionId: command.chatId,
+            traceId: "chat:" + command.id,
+            input: command.text,
+          },
+          async () => {
+            try {
+              if (settings.codexThreadId) {
+                const delivery = await queueCodexChatCommand({
+                  settings,
+                  client,
+                  identity,
+                  command,
+                  claim,
+                  signal: commandSignal,
+                });
                 emit({
-                  type: `chat_${run.state}`,
-                  data: { chatId: command.chatId, commandId: command.id },
+                  type: delivery.state === "uncertain" ? "chat_uncertain" : "chat_queued",
+                  data: {
+                    chatId: command.chatId,
+                    commandId: command.id,
+                    threadId: settings.codexThreadId,
+                  },
                 });
-                break;
+                // Native tools perform the work; only the native callback completes it.
+                // Keep the lease alive while independently processing inbound peer replies.
+                while (!commandSignal.aborted) {
+                  const { run } = await retryNetwork(
+                    () => readCodexChatCommand(client, command, commandSignal),
+                    { signal: commandSignal, delayMs: settings.pollMs, onRetry: report },
+                  );
+                  if (["completed", "failed", "canceled"].includes(run.state)) {
+                    emit({
+                      type: `chat_${run.state}`,
+                      data: { chatId: command.chatId, commandId: command.id },
+                    });
+                    break;
+                  }
+                  await pause(Math.max(settings.pollMs, 1000), commandSignal);
+                }
+                return;
               }
-              await pause(Math.max(settings.pollMs, 1000), commandSignal);
-            }
-            continue;
-          }
-          await settings.inspector.assertAllowed("input", command.text, { signal: commandSignal });
-          const explicitRequest = parseAskCommand(command.text);
-          let text;
-          if (explicitRequest) {
-            text = await explicitAskReply({
-              client,
-              identity,
-              journal,
-              command,
-              request: explicitRequest,
-              settings,
-              signal: commandSignal,
-            });
-          } else {
-            const chatEvents = [];
-            let chatCursor = "0";
-            for (let page = 0; page < 100; page++) {
-              const detail = await client.chats.get(command.chatId, {
+              await settings.inspector.assertAllowed("input", command.text, {
                 signal: commandSignal,
-                limit: 100,
-                ...(chatCursor ? { cursor: chatCursor } : {}),
               });
-              chatEvents.push(...detail.events);
-              await settings.inspector.assertAllowed("input", detail.events, {
+              const explicitRequest = parseAskCommand(command.text);
+              let text;
+              if (explicitRequest) {
+                text = await explicitAskReply({
+                  client,
+                  identity,
+                  journal,
+                  command,
+                  request: explicitRequest,
+                  settings,
+                  signal: commandSignal,
+                });
+              } else {
+                const chatEvents = [];
+                let chatCursor = "0";
+                for (let page = 0; page < 100; page++) {
+                  const detail = await client.chats.get(command.chatId, {
+                    signal: commandSignal,
+                    limit: 100,
+                    ...(chatCursor ? { cursor: chatCursor } : {}),
+                  });
+                  chatEvents.push(...detail.events);
+                  await settings.inspector.assertAllowed("input", detail.events, {
+                    signal: commandSignal,
+                  });
+                  if (
+                    chatEvents.some((event) => event.id === command.commandEventId) ||
+                    !detail.hasMore
+                  )
+                    break;
+                  chatEvents.splice(0, Math.max(0, chatEvents.length - 24));
+                  chatCursor = detail.nextCursor;
+                }
+                text = await generate({
+                  client,
+                  identity,
+                  journal,
+                  key: `chat:${command.commandEventId}`,
+                  text: command.text,
+                  history: boundedHistory(chatEvents, command.commandEventId),
+                  settings: { ...settings, observabilitySessionId: command.chatId },
+                  signal: commandSignal,
+                  allowTools: true,
+                });
+              }
+              observationOutcome({ output: text });
+              if (leaseError) throw leaseError;
+              await settings.inspector.assertAllowed("output", text, {
+                task: command.text,
                 signal: commandSignal,
               });
-              if (
-                chatEvents.some((event) => event.id === command.commandEventId) ||
-                !detail.hasMore
-              )
-                break;
-              chatEvents.splice(0, Math.max(0, chatEvents.length - 24));
-              chatCursor = detail.nextCursor;
-            }
-            text = await generate({
-              client,
-              identity,
-              journal,
-              key: `chat:${command.commandEventId}`,
-              text: command.text,
-              history: boundedHistory(chatEvents, command.commandEventId),
-              settings,
-              signal: commandSignal,
-              allowTools: true,
-            });
-          }
-          if (leaseError) throw leaseError;
-          await settings.inspector.assertAllowed("output", text, {
-            task: command.text,
-            signal: commandSignal,
-          });
-          await client.chatCommands.complete(
-            command.id,
-            { leaseToken: claim.leaseToken, text },
-            {
-              idempotencyKey: operationKey(identity.id, "complete", command.commandEventId),
-              signal: commandSignal,
-            },
-          );
-          emit({ type: "chat_completed", data: { chatId: command.chatId, commandId: command.id } });
-          leaseStop.abort();
-          if (!settings.connectionOnly && !explicitRequest)
-            await learnAgentMemory({
-              client,
-              agentId: identity.id,
-              key: command.commandEventId,
-              turn: { user: command.text, assistant: text },
-              journal,
-              inspector: settings.inspector,
-              signal,
-              emit,
-              generate: async (messages) =>
-                (
-                  await modelReply({
-                    origin: settings.origin,
-                    provider: settings.provider,
-                    qwenUrl: settings.qwenUrl,
-                    nimUrl: settings.nimUrl,
-                    model: settings.model,
-                    messages,
-                    tools: [],
-                    signal,
-                    timeoutMs: Math.min(settings.modelTimeoutMs ?? 900000, 30000),
-                    adapterSettings: settings.adapterSettings,
-                    diagnosticDirectory: settings.diagnosticDirectory,
-                  })
-                ).content,
-            });
-        } catch (error) {
-          // Native queue outcomes remain durable, never converted into a false failure
-          // or replayed after an uncertain dispatch. Unsent rate-limited work can retry.
-          if (!settings.codexThreadId && !signal.aborted && !leaseError) {
-            try {
-              await client.chatCommands.fail(
+              await client.chatCommands.complete(
                 command.id,
+                { leaseToken: claim.leaseToken, text },
                 {
-                  leaseToken: claim.leaseToken,
-                  error: [
-                    "AGENT_REPLY_TIMEOUT",
-                    "INVALID_ASK",
-                    "TOOL_NOT_ALLOWED",
-                    "INVALID_PEER_REPLY",
-                  ].includes(error.code)
-                    ? error.message
-                    : "The agent runtime could not complete this request. Check the runtime and send a new message to retry.",
+                  idempotencyKey: operationKey(identity.id, "complete", command.commandEventId),
+                  signal: commandSignal,
                 },
-                { signal },
               );
-            } catch (failure) {
-              report(failure, { commandId: command.id });
+              emit({
+                type: "chat_completed",
+                data: { chatId: command.chatId, commandId: command.id },
+              });
+              leaseStop.abort();
+              if (!settings.connectionOnly && !explicitRequest)
+                await learnAgentMemory({
+                  client,
+                  agentId: identity.id,
+                  key: command.commandEventId,
+                  turn: { user: command.text, assistant: text },
+                  journal,
+                  inspector: settings.inspector,
+                  signal,
+                  emit,
+                  generate: async (messages) =>
+                    (
+                      await modelReply({
+                        origin: settings.origin,
+                        provider: settings.provider,
+                        qwenUrl: settings.qwenUrl,
+                        nimUrl: settings.nimUrl,
+                        model: settings.model,
+                        messages,
+                        tools: [],
+                        signal,
+                        timeoutMs: Math.min(settings.modelTimeoutMs ?? 900000, 30000),
+                        adapterSettings: settings.adapterSettings,
+                        diagnosticDirectory: settings.diagnosticDirectory,
+                      })
+                    ).content,
+                });
+            } catch (error) {
+              observationOutcome({ error });
+              // Native queue outcomes remain durable, never converted into a false failure
+              // or replayed after an uncertain dispatch. Unsent rate-limited work can retry.
+              if (!settings.codexThreadId && !signal.aborted && !leaseError) {
+                try {
+                  await client.chatCommands.fail(
+                    command.id,
+                    {
+                      leaseToken: claim.leaseToken,
+                      error: [
+                        "AGENT_REPLY_TIMEOUT",
+                        "INVALID_ASK",
+                        "TOOL_NOT_ALLOWED",
+                        "INVALID_PEER_REPLY",
+                      ].includes(error.code)
+                        ? error.message
+                        : "The agent runtime could not complete this request. Check the runtime and send a new message to retry.",
+                    },
+                    { signal },
+                  );
+                } catch (failure) {
+                  report(failure, { commandId: command.id });
+                }
+              }
+              if (!signal.aborted) report(leaseError ?? error, { commandId: command.id });
+            } finally {
+              leaseStop.abort();
+              await renewal;
             }
-          }
-          if (!signal.aborted) report(leaseError ?? error, { commandId: command.id });
-        } finally {
-          leaseStop.abort();
-          await renewal;
-        }
+          },
+        );
       }
       if (!settings.once && !signal.aborted) await pause(settings.pollMs, signal);
     } while (!signal.aborted && !settings.once);