threadwire 0.1.14 → 0.1.17

package/src/telegram-ingress/core.js

@@ -2,6 +2,7 @@
 
 import {createHash, randomUUID, timingSafeEqual} from "node:crypto"
 import {abortable} from "../absolute-deadline.js"
+import {DelegatedResultAdmission} from "../delegated-result-admission.js"
 import {evidenceTelegramDestination} from "../evidence-store.js"
 import {NoticeQueue} from "../notice-queue.js"
 import {createFetchTransport} from "../notifiers/fetch-transport.js"
@@ -136,6 +137,17 @@ export async function dispatchWorker(job, config, dependencies = {}) {
   let control
   /** @type {Awaited<ReturnType<import("../evidence-store.js").EvidenceStore["createArtifact"]>> | undefined} */
   let evidence
+  /** @type {Awaited<ReturnType<import("../evidence-store.js").EvidenceStore["createArtifact"]>> | undefined} */
+  let resultEvidence
+  /** @type {{destinationId: string, runId: string} | undefined} */
+  let evidenceOwner
+  let resultEvidenceFinalized = false
+  /** @type {unknown} */
+  let evidenceError
+  /** @type {unknown} */
+  let deliveryError
+  const deliveryIdentity = randomUUID()
+  const admission = new DelegatedResultAdmission({output: {write: () => {}}})
   try {
     provider = createProviderImpl(job.provider, [], job.prompt, undefined, providerEnvironment)
     const transport = createFetchTransportImpl(config.botToken, undefined, config.telegramRequestTimeoutMs)
@@ -147,18 +159,99 @@ export async function dispatchWorker(job, config, dependencies = {}) {
       NoticeQueueClass: NoticeQueueImpl,
       RelayClass: RelayImpl
     })
-    const evidenceOwner = dependencies.evidenceStore?.createOwnerScope({
+    evidenceOwner = dependencies.evidenceStore?.createOwnerScope({
       destinationId: evidenceTelegramDestination(job.target, job.senderId),
-      runId: randomUUID()
-    })
-    evidence = evidenceOwner === undefined ? undefined : await dependencies.evidenceStore?.createArtifact(evidenceOwner, {
-      contentType: "text/plain; charset=utf-8",
-      redactions: await collectEvidenceRedactions(dependencies.providerEnvironment ?? {})
+      runId: deliveryIdentity
     })
+    if (evidenceOwner !== undefined) {
+      const redactions = await collectEvidenceRedactions(dependencies.providerEnvironment ?? {})
+      evidence = await dependencies.evidenceStore?.createArtifact(evidenceOwner, {
+        contentType: "text/plain; charset=utf-8",
+        redactions
+      })
+      try {
+        resultEvidence = await dependencies.evidenceStore?.createArtifact(evidenceOwner, {
+          contentType: "application/json",
+          redactions,
+          delivery: {identity: deliveryIdentity, state: "delivery_pending"}
+        })
+      } catch (error) {
+        await evidence?.abort()
+        evidence = undefined
+        throw error
+      }
+    }
   } catch (error) {
     earlyKimiPreflight?.deadline?.close()
     throw error
   }
+  /** @param {import("../types.js").WorkerEvent} event */
+  const acceptEvent = async (event) => {
+    if (event.type === "text-delta") admission.acceptConclusionEvent(event)
+    try {
+      await control.accept(event)
+    } catch (error) {
+      deliveryError ??= error
+    }
+  }
+  /** @param {unknown} record */
+  const acceptRecord = (record) => {
+    const id = job.provider === "kimi" && isolatedRuntimeClient !== undefined
+      ? kimiSessionEnvelopeId(record)
+      : provider.sessionId(record)
+    if (id !== undefined) {
+      admission.setContinuationHandle(id)
+      dependencies.activity?.recordSession(provider.name, id)
+    }
+  }
+  /** @param {number} exitCode */
+  const finalizeEvidence = async (exitCode) => {
+    if (resultEvidence === undefined || resultEvidenceFinalized) return
+    let rawEvidenceReady = false
+    if (evidence !== undefined) {
+      try {
+        await evidence.finalize()
+        rawEvidenceReady = true
+      } catch (error) {
+        await evidence.abort()
+        evidenceError ??= error
+      }
+    }
+    admission.addArtifactHandle(resultEvidence.handle)
+    if (rawEvidenceReady) admission.addArtifactHandle(/** @type {NonNullable<typeof evidence>} */ (evidence).handle)
+    const terminal = {
+      state: /** @type {"completed" | "failed"} */ (exitCode === 0 ? "completed" : "failed"),
+      exitCode
+    }
+    await resultEvidence.append("provider-result", `${JSON.stringify(admission.createProviderResult(terminal))}\n`)
+    await resultEvidence.finalize()
+    resultEvidenceFinalized = true
+  }
+  const finishDelivery = async () => {
+    if (resultEvidence !== undefined && deliveryError === undefined) {
+      try {
+        await control.acceptEvidenceHandle(resultEvidence.handle)
+      } catch (error) {
+        deliveryError ??= error
+      }
+    }
+    try {
+      await control.close()
+    } catch (error) {
+      deliveryError ??= error
+    }
+    if (
+      resultEvidenceFinalized && resultEvidence !== undefined && evidenceOwner !== undefined
+      && dependencies.evidenceStore !== undefined
+    ) {
+      await dependencies.evidenceStore.recordDelivery(
+        evidenceOwner,
+        resultEvidence.handle,
+        deliveryError === undefined ? "delivery_succeeded" : "delivery_failed"
+      )
+    }
+    if (deliveryError !== undefined) throw deliveryError
+  }
   let evidenceTransferred = false
   try {
     await evidence?.append("prompt", `prompt\n${job.prompt}\nprovider-stream\n`)
@@ -181,33 +274,56 @@ export async function dispatchWorker(job, config, dependencies = {}) {
           prompt: job.prompt,
           providerArguments: [],
           ...(preflight.deadline === undefined ? {} : {deadline: preflight.deadline}),
-          onEvent: async (event) => control.accept(event),
-          onRecord: async (record) => {
-            const id = job.provider === "kimi" ? kimiSessionEnvelopeId(record) : provider.sessionId(record)
-            if (id !== undefined) dependencies.activity?.recordSession(provider.name, id)
-          },
+          onEvent: acceptEvent,
+          onRecord: async (record) => acceptRecord(record),
           ...(job.provider === "kimi" ? {} : {
             onStdoutChunk: (chunk) => evidence?.append("provider-stdout", chunk),
             onStderrChunk: (chunk) => evidence?.append("provider-stderr", chunk)
           })
         })
-        let exitCode
+        let exitCode = 2
+        /** @type {Error | undefined} */
+        let executionError
         try {
           exitCode = await abortable(isolatedRun.completion, preflight.deadline?.signal)
         } catch (error) {
-          await isolatedRun.cancel(error instanceof Error ? error : new Error("Isolated runtime run failed"))
-          throw error
+          executionError = error instanceof Error ? error : new Error("Isolated runtime run failed")
+          try {
+            await isolatedRun.cancel(executionError)
+          } catch (cancelError) {
+            executionError = new AggregateError(
+              [executionError, cancelError],
+              "Isolated runtime failed and cleanup could not be confirmed"
+            )
+          }
         }
-        if (exitCode !== 0) throw new Error(`Isolated ${job.provider === "kimi" ? "Kimi" : "Codex"} worker exited with status ${exitCode}`)
-        await control.close()
-        if (evidence !== undefined) {
-          await evidence.finalize()
-          await sender.send(`Evidence: ${evidence.handle}`)
+        await finalizeEvidence(exitCode)
+        let propagatedExecutionError = executionError
+        try {
+          await finishDelivery()
+        } catch (error) {
+          if (propagatedExecutionError === undefined) throw error
+          propagatedExecutionError = new AggregateError(
+            [propagatedExecutionError, error],
+            propagatedExecutionError.message
+          )
         }
+        if (evidenceError !== undefined) {
+          if (propagatedExecutionError === undefined) throw evidenceError
+          propagatedExecutionError = new AggregateError(
+            [propagatedExecutionError, evidenceError],
+            propagatedExecutionError.message
+          )
+        }
+        if (propagatedExecutionError !== undefined) throw propagatedExecutionError
+        if (exitCode !== 0) throw new Error(`Isolated ${job.provider === "kimi" ? "Kimi" : "Codex"} worker exited with status ${exitCode}`)
         return
       } catch (error) {
         await control.close().catch(() => {})
-        await evidence?.abort()
+        if (!resultEvidenceFinalized) {
+          await evidence?.abort()
+          await resultEvidence?.abort()
+        }
         throw error
       } finally {
         isolatedDeadline?.close()
@@ -260,15 +376,12 @@ export async function dispatchWorker(job, config, dependencies = {}) {
       provider: provider.name,
       parse: provider.parse,
       ...(provider.completion === undefined ? {} : {completion: provider.completion}),
-      onEvent: async (event) => control.accept(event),
+      onEvent: acceptEvent,
       onSpawn: (pid) => {
         if (pid !== undefined) dependencies.activity?.recordStarted(provider.name, pid)
         spawnGate.resolve()
       },
-      onRecord: async (record) => {
-        const id = provider.sessionId(record)
-        if (id !== undefined) dependencies.activity?.recordSession(provider.name, id)
-      },
+      onRecord: async (record) => acceptRecord(record),
       onStdoutChunk: (chunk) => evidence?.append("provider-stdout", chunk),
       onStderrChunk: (chunk) => evidence?.append("provider-stderr", chunk)
     })
@@ -278,49 +391,61 @@ export async function dispatchWorker(job, config, dependencies = {}) {
     // reported. Always notify settlement so concurrency slots are released.
     evidenceTransferred = true
     void (async () => {
+      let exitCode = 2
       /** @type {Error | undefined} */
-      let nativeKimiExitFailure
+      let failure
+      let failureReported = false
       try {
         try {
-          const exitCode = await running
+          exitCode = await running
           if (job.provider === "kimi" && spawnGate.settled) {
-            nativeKimiExitFailure = exitCode === 0 ? undefined : new Error(`Kimi worker exited with status ${exitCode}`)
-            await control.accept({
+            await acceptEvent({
               type: "lifecycle",
               phase: exitCode === 0 ? "completed" : "failed",
               summary: exitCode === 0 ? "Kimi worker completed" : "Kimi worker failed"
             })
-            if (nativeKimiExitFailure !== undefined) throw nativeKimiExitFailure
+            if (exitCode !== 0) failure = new Error(`Kimi worker exited with status ${exitCode}`)
           }
         } catch (error) {
-          const failure = nativeKimiExitFailure ?? (error instanceof Error ? error : new Error(safeFailureMessage(error)))
-          if (spawnGate.settled) {
-            reportBackgroundFailure(failure)
-          } else {
-            spawnGate.reject(failure)
-          }
-          try {
-            await control.close()
-          } catch {
-          /* Best-effort close after worker failure. */
-          }
-          return
+          failure = error instanceof Error ? error : new Error(safeFailureMessage(error))
+        }
+        if (!spawnGate.settled && failure === undefined) {
+          exitCode = 2
+          failure = new Error("Worker exited before spawn was observed")
         }
 
         try {
-          await control.close()
+          await finalizeEvidence(exitCode)
         } catch (error) {
-          if (spawnGate.settled) {
-            reportBackgroundFailure(error)
-          } else {
-            spawnGate.reject(error instanceof Error ? error : new Error(safeFailureMessage(error)))
-          }
-          return
+          await evidence?.abort()
+          await resultEvidence?.abort()
+          failure ??= error instanceof Error ? error : new Error(safeFailureMessage(error))
+        }
+        if (evidenceError !== undefined) {
+          failure ??= evidenceError instanceof Error ? evidenceError : new Error(safeFailureMessage(evidenceError))
         }
 
-        if (!spawnGate.settled) {
-          spawnGate.reject(new Error("Worker exited before spawn was observed"))
+        if (spawnGate.settled && failure !== undefined) {
+          reportBackgroundFailure(failure)
+          failureReported = true
         }
+
+        if (evidence === undefined || resultEvidenceFinalized) {
+          try {
+            await finishDelivery()
+          } catch (error) {
+            failure ??= error instanceof Error ? error : new Error(safeFailureMessage(error))
+          }
+        } else {
+          try {
+            await control.close()
+          } catch (error) {
+            failure ??= error instanceof Error ? error : new Error(safeFailureMessage(error))
+          }
+        }
+
+        if (!spawnGate.settled) spawnGate.reject(/** @type {Error} */ (failure))
+        else if (failure !== undefined && !failureReported) reportBackgroundFailure(failure)
       } catch (error) {
         if (spawnGate.settled) {
           reportBackgroundFailure(error)
@@ -328,15 +453,6 @@ export async function dispatchWorker(job, config, dependencies = {}) {
           spawnGate.reject(error instanceof Error ? error : new Error(safeFailureMessage(error)))
         }
       } finally {
-        if (evidence !== undefined) {
-          try {
-            await evidence.finalize()
-            await sender.send(`Evidence: ${evidence.handle}`)
-          } catch (error) {
-            await evidence.abort()
-            reportBackgroundFailure(error)
-          }
-        }
         notifySettled()
       }
     })()
@@ -344,7 +460,10 @@ export async function dispatchWorker(job, config, dependencies = {}) {
     await spawned
   } finally {
     try {
-      if (!evidenceTransferred) await evidence?.abort()
+      if (!evidenceTransferred) {
+        await evidence?.abort()
+        await resultEvidence?.abort()
+      }
     } finally {
       earlyKimiPreflight?.deadline?.close()
     }

package/src/worker-control.js

@@ -98,6 +98,17 @@ export class WorkerControl {
     })
   }
 
+  /** @param {string} handle */
+  acceptEvidenceHandle(handle) {
+    if (!/^evidence_[A-Za-z0-9_-]{43}$/u.test(handle)) throw new Error("Invalid evidence handle")
+    return this.enqueueOperation(async () => {
+      this.ensureHealthy()
+      this.renderedNotices.push({text: `Evidence: ${handle}`})
+      await this.flushLoop(false)
+      this.ensureHealthy()
+    })
+  }
+
   close() {
     if (!this.closePromise) {
       this.closePromise = this.enqueueOperation(async () => {