wtagent 0.3.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +163 -9
  2. package/package.json +9 -8
  3. package/src/artifacts/artifact-store.js +154 -0
  4. package/src/audio/music-artifact.js +66 -0
  5. package/src/audio/native-music-receiver.js +91 -0
  6. package/src/browser/base-web-adapter.js +2418 -201
  7. package/src/browser/cdp-browser.js +223 -25
  8. package/src/browser/cdp-state.js +8 -5
  9. package/src/browser/chatgpt-dom.js +179 -0
  10. package/src/browser/chatgpt-web-adapter.js +831 -69
  11. package/src/browser/claude-web-adapter.js +13 -0
  12. package/src/browser/fake-web-model-adapter.js +152 -3
  13. package/src/browser/gemini-web-adapter.js +71 -20
  14. package/src/browser/glm-web-adapter.js +2 -2
  15. package/src/browser/grok-web-adapter.js +35 -3
  16. package/src/browser/rendered-text.js +11 -0
  17. package/src/cli/i18n.js +50 -20
  18. package/src/cli/main.js +124 -40
  19. package/src/cli/prompt-input.js +68 -0
  20. package/src/image/browser-image-driver.js +238 -0
  21. package/src/image/browser-image-session-pool.js +96 -0
  22. package/src/image/image-generation-service.js +101 -0
  23. package/src/image/image-provider-driver.js +22 -0
  24. package/src/image/native-image-receiver.js +66 -0
  25. package/src/image/provider-registry.js +23 -0
  26. package/src/image/providers/chatgpt-image-driver.js +211 -0
  27. package/src/image/providers/gemini-image-driver.js +100 -0
  28. package/src/image/providers/grok-image-driver.js +193 -0
  29. package/src/platform/command-launcher.js +1 -1
  30. package/src/platform/paths.js +19 -0
  31. package/src/platform/windows-diagnostics.js +9 -16
  32. package/src/policy/path-guard.js +5 -1
  33. package/src/policy/policy-engine.js +10 -5
  34. package/src/protocol/envelope.js +74 -0
  35. package/src/protocol/markers.js +16 -0
  36. package/src/protocol/prompt-builder.js +12 -9
  37. package/src/protocol/xml-protocol.js +36 -18
  38. package/src/runtime/agent-runtime.js +1383 -238
  39. package/src/session/agent-session.js +1047 -121
  40. package/src/tools/default-tools.js +1 -1
  41. package/src/tools/image-tools.js +92 -0
  42. package/src/tools/registry.js +13 -1
  43. package/docs/technical-design.md +0 -866
@@ -1,13 +1,16 @@
1
- import { createHash } from "node:crypto";
1
+ import { createHash, randomUUID } from "node:crypto";
2
2
  import {
3
3
  cdata,
4
4
  extractTrailingProse,
5
5
  parseAgentResponse,
6
6
  serializeProtocolError,
7
7
  serializeToolResult,
8
- stripUiNoiseLines,
9
8
  } from "../protocol/xml-protocol.js";
10
- import { appendSystemReminder } from "../protocol/markers.js";
9
+ import {
10
+ appendSystemReminder,
11
+ DEFAULT_SYSTEM_REMINDER,
12
+ NATIVE_IMAGE_SYSTEM_REMINDER,
13
+ } from "../protocol/markers.js";
11
14
  import {
12
15
  buildBootstrapPrompt,
13
16
  buildResumePrompt,
@@ -16,17 +19,22 @@ import {
16
19
  assistantMessage,
17
20
  functionCall,
18
21
  functionCallOutput,
22
+ messageText,
19
23
  toolResultOutput,
20
24
  userMessage,
21
25
  } from "../session/canonical-transcript.js";
22
26
  import { DEFAULT_LIMITS } from "../shared/limits.js";
27
+ import { MUSIC_SYSTEM_REMINDER } from "../audio/native-music-receiver.js";
23
28
  import { utf8ByteLength } from "../shared/text-budget.js";
24
29
  import {
25
30
  BrowserAdapterError,
26
31
  ProtocolError,
27
32
  ToolValidationError,
28
33
  } from "../shared/errors.js";
29
- import { isConnectionLostError } from "../browser/base-web-adapter.js";
34
+ import {
35
+ isConnectionLostError,
36
+ isValidOutboundCorrelationId,
37
+ } from "../browser/base-web-adapter.js";
30
38
  import { isUsageLimitNotice } from "../shared/usage-limit.js";
31
39
 
32
40
  const EMPTY_ASSISTANT_CONTINUE_MESSAGE =
@@ -44,6 +52,11 @@ const GENERATION_FAILED_CONTINUE_MESSAGE =
44
52
  + "Retry the immediately preceding task from the existing conversation context. Do not "
45
53
  + "repeat any local tool operation whose result is already present. Reply using the "
46
54
  + "required <agent_response> XML protocol.";
55
+ const OUTBOUND_CORRELATION_PREFIX =
56
+ " Opaque WTAgent transport correlation ID (do not repeat): ";
57
+ const OUTBOUND_CORRELATION_RESERVE_BYTES = utf8ByteLength(
58
+ `${OUTBOUND_CORRELATION_PREFIX}${"0".repeat(36)}.`,
59
+ );
47
60
 
48
61
  function canonicalize(value) {
49
62
  if (Array.isArray(value)) {
@@ -145,6 +158,8 @@ export class AgentRuntime {
145
158
  approval,
146
159
  onEvent,
147
160
  postAuthSetup = null,
161
+ nativeImageReceiver = null,
162
+ nativeMusicReceiver = null,
148
163
  limits = DEFAULT_LIMITS,
149
164
  }) {
150
165
  this.adapter = adapter;
@@ -154,7 +169,14 @@ export class AgentRuntime {
154
169
  this.approval = approval;
155
170
  this.onEvent = onEvent;
156
171
  this.postAuthSetup = postAuthSetup;
172
+ this.nativeImageReceiver = nativeImageReceiver;
173
+ this.nativeMusicReceiver = nativeMusicReceiver;
157
174
  this.limits = limits;
175
+ this.conversationIdentityQueue = Promise.resolve();
176
+ this.deferConversationIdentityPersistence = false;
177
+ this.adapter.setConversationIdentityListener?.((identity) =>
178
+ this.#queueConversationIdentity(identity)
179
+ );
158
180
  }
159
181
 
160
182
  async emit(type, payload = {}) {
@@ -163,60 +185,438 @@ export class AgentRuntime {
163
185
  return event;
164
186
  }
165
187
 
166
- async sendMessage(text, { files = [], maxBytes = null } = {}) {
167
- const message = appendSystemReminder(text);
188
+ #assertProviderReadyForMessage() {
189
+ if (this.session.state.pendingAuxiliaryTurn != null) {
190
+ throw new BrowserAdapterError(
191
+ "The provider operation has no verified assistant boundary. No further message was sent.",
192
+ { code: "AUXILIARY_TURN_RECOVERY_REQUIRED", recoverable: false },
193
+ );
194
+ }
195
+ if (this.session.state.pendingOutbound != null) {
196
+ throw new BrowserAdapterError(
197
+ "A previous provider message has not been reconciled. No further message was sent.",
198
+ { code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
199
+ );
200
+ }
201
+ }
202
+
203
+ async sendMessage(text, {
204
+ files = [],
205
+ maxBytes = null,
206
+ allowFreshReconnect = false,
207
+ allowAssistantContinuation = false,
208
+ requireAttachments = false,
209
+ outboundKind = "runtime_message",
210
+ transcriptItems = [],
211
+ runtimeTurn = null,
212
+ pendingToolAcknowledgement = null,
213
+ } = {}) {
214
+ this.#assertProviderReadyForMessage();
215
+ await this.nativeMusicReceiver?.prepare();
216
+ const outboundId = randomUUID();
217
+ const reminder = this.nativeMusicReceiver ? MUSIC_SYSTEM_REMINDER
218
+ : this.nativeImageReceiver ? NATIVE_IMAGE_SYSTEM_REMINDER : DEFAULT_SYSTEM_REMINDER;
219
+ const message = appendSystemReminder(
220
+ text,
221
+ `${reminder}${OUTBOUND_CORRELATION_PREFIX}${outboundId}.`,
222
+ );
223
+ const pendingOutbound = {
224
+ kind: outboundKind,
225
+ outboundId,
226
+ messageHash: createHash("sha256").update(message).digest("hex"),
227
+ preparedAt: new Date().toISOString(),
228
+ transcriptItems,
229
+ runtimeTurn: Number.isSafeInteger(runtimeTurn) ? runtimeTurn : null,
230
+ pendingToolAcknowledgement,
231
+ conversationUrl: this.session.state.conversationUrl ?? null,
232
+ conversationTargetId: this.session.state.conversationTargetId ?? null,
233
+ };
234
+ await this.#queueSessionUpdate({ pendingOutbound });
168
235
  try {
169
- await this.#sendMessageWithReconnect(message, { files, maxBytes });
236
+ const sendResult = await this.#sendMessageWithReconnect(message, {
237
+ files,
238
+ maxBytes,
239
+ allowFreshReconnect,
240
+ allowAssistantContinuation,
241
+ requireAttachments,
242
+ outboundId,
243
+ });
244
+ const handoff = await this.#queueSessionOperation(() => (
245
+ this.session.commitPendingOutboundHandoff({
246
+ outboundId,
247
+ handoff: {
248
+ runtimeTurn: pendingOutbound.runtimeTurn
249
+ ?? this.session.state.turn,
250
+ conversationUrl: sendResult?.conversationUrl
251
+ ?? this.session.state.conversationUrl,
252
+ conversationTargetId: sendResult?.conversationTargetId
253
+ ?? this.session.state.conversationTargetId,
254
+ userMessageId: sendResult?.userMessageId ?? null,
255
+ userTurn: sendResult?.userTurn ?? null,
256
+ assistantBaseline: sendResult?.assistantBaseline ?? null,
257
+ preOutboundMarkerIds: sendResult?.preOutboundMarkerIds ?? [],
258
+ pendingToolAcknowledgement,
259
+ },
260
+ })
261
+ ));
262
+ return { ...sendResult, pendingAssistantTurn: handoff };
263
+ } catch (error) {
264
+ const sendStatus = this.adapter.getLastSendStatus?.() ?? "commit-unknown";
265
+ const nextPending = sendStatus === "not-submitted"
266
+ ? null
267
+ : {
268
+ ...pendingOutbound,
269
+ status: "commit-unknown",
270
+ failedAt: new Date().toISOString(),
271
+ };
272
+ await this.#queueSessionOperation(async () => {
273
+ if (this.session.state.pendingOutbound?.outboundId === outboundId) {
274
+ await this.session.update({ pendingOutbound: nextPending });
275
+ }
276
+ }).catch(() => {});
277
+ throw error;
170
278
  } finally {
171
- const conversationUrl = await this.adapter.getConversationUrl()
279
+ // URL observation must never replace the send result. Top-frame navigation
280
+ // is persisted eagerly; this is only a best-effort final checkpoint.
281
+ const identity = await this.#syncConversationIdentity({
282
+ suppressErrors: true,
283
+ });
284
+ await this.#refreshPendingOutboundIdentity(outboundId, identity)
172
285
  .catch(() => null);
286
+ }
287
+ }
288
+
289
+ async #persistConversationIdentity(identity, { allowFresh = false } = {}) {
290
+ const conversationUrl = identity?.conversationUrl ?? null;
291
+ const kind = identity?.kind
292
+ ?? (conversationUrl && this.adapter.classifyConversationUrl?.(conversationUrl));
293
+ if (
294
+ !conversationUrl
295
+ || (
296
+ !["restorable", "provisional"].includes(kind)
297
+ && !(allowFresh && kind === "fresh")
298
+ )
299
+ ) {
300
+ return null;
301
+ }
302
+
303
+ const targetId = typeof identity?.targetId === "string"
304
+ && identity.targetId
305
+ ? identity.targetId
306
+ : null;
307
+ const patch = {};
308
+ if (conversationUrl !== this.session.state.conversationUrl) {
309
+ patch.conversationUrl = conversationUrl;
310
+ }
311
+ if (
312
+ targetId
313
+ && targetId !== this.session.state.conversationTargetId
314
+ ) {
315
+ patch.conversationTargetId = targetId;
316
+ }
317
+ if (Object.keys(patch).length > 0) {
318
+ await this.session.update(patch);
319
+ }
320
+ return {
321
+ conversationUrl,
322
+ targetId: targetId ?? this.session.state.conversationTargetId ?? null,
323
+ kind,
324
+ };
325
+ }
326
+
327
+ #queueConversationIdentity(identity, options = {}) {
328
+ if (this.deferConversationIdentityPersistence) {
329
+ return Promise.resolve(null);
330
+ }
331
+ const operation = this.conversationIdentityQueue
332
+ .catch(() => null)
333
+ .then(() => this.#persistConversationIdentity(identity, options));
334
+ this.conversationIdentityQueue = operation;
335
+ return operation;
336
+ }
337
+
338
+ #queueSessionOperation(callback) {
339
+ const operation = this.conversationIdentityQueue
340
+ .catch(() => null)
341
+ .then(callback);
342
+ this.conversationIdentityQueue = operation;
343
+ return operation;
344
+ }
345
+
346
+ #queueSessionUpdate(patch) {
347
+ return this.#queueSessionOperation(() => this.session.update(patch));
348
+ }
349
+
350
+ async #refreshPendingOutboundIdentity(
351
+ outboundId,
352
+ identity = null,
353
+ { required = false } = {},
354
+ ) {
355
+ return await this.#queueSessionOperation(async () => {
356
+ const current = this.session.state.pendingOutbound;
357
+ if (!current || current.outboundId !== outboundId) {
358
+ if (required) {
359
+ throw new BrowserAdapterError(
360
+ "The pending outbound changed before its reconnect target could be checkpointed.",
361
+ { code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
362
+ );
363
+ }
364
+ return null;
365
+ }
366
+ const conversationUrl = identity?.conversationUrl
367
+ ?? this.session.state.conversationUrl
368
+ ?? current.conversationUrl
369
+ ?? null;
370
+ const conversationTargetId = identity?.targetId
371
+ ?? identity?.conversationTargetId
372
+ ?? this.session.state.conversationTargetId
373
+ ?? current.conversationTargetId
374
+ ?? null;
173
375
  if (
174
- conversationUrl
175
- && conversationUrl !== this.session.state.conversationUrl
376
+ conversationUrl === current.conversationUrl
377
+ && conversationTargetId === current.conversationTargetId
176
378
  ) {
177
- await this.session.update({ conversationUrl });
379
+ return structuredClone(current);
380
+ }
381
+ const refreshed = {
382
+ ...current,
383
+ conversationUrl,
384
+ conversationTargetId,
385
+ };
386
+ await this.session.update({
387
+ pendingOutbound: refreshed,
388
+ conversationUrl,
389
+ conversationTargetId,
390
+ });
391
+ return structuredClone(refreshed);
392
+ });
393
+ }
394
+
395
+ async #syncConversationIdentity({
396
+ suppressErrors = false,
397
+ allowFresh = false,
398
+ identity = null,
399
+ } = {}) {
400
+ // Observe and persist inside one queue entry. If a navigation callback fires
401
+ // while observation awaits the adapter, it queues after this operation and
402
+ // remains authoritative instead of being overwritten by a stale snapshot.
403
+ const operation = this.conversationIdentityQueue
404
+ .catch(() => null)
405
+ .then(async () => {
406
+ let observed = identity;
407
+ if (!observed) {
408
+ if (typeof this.adapter.getConversationIdentity === "function") {
409
+ observed = await this.adapter.getConversationIdentity();
410
+ } else {
411
+ const conversationUrl = await this.adapter.getConversationUrl();
412
+ observed = {
413
+ conversationUrl,
414
+ targetId: null,
415
+ kind: this.adapter.classifyConversationUrl?.(conversationUrl)
416
+ ?? (/\/c\//.test(new URL(conversationUrl).pathname)
417
+ ? "restorable"
418
+ : "fresh"),
419
+ };
420
+ }
421
+ }
422
+ return await this.#persistConversationIdentity(observed, { allowFresh });
423
+ });
424
+ this.conversationIdentityQueue = operation;
425
+ try {
426
+ return await operation;
427
+ } catch (error) {
428
+ if (!suppressErrors) {
429
+ throw error;
178
430
  }
431
+ return null;
179
432
  }
180
433
  }
181
434
 
182
- // Sends once; if the browser connection died in between (e.g. the Mac slept
183
- // while a tool was running), reconnects to the still-alive Chrome, restores
184
- // the conversation, and retries once. A send that never rendered (ChatGPT
185
- // did not register the message) is also retried once — the deterministic
186
- // message is simply sent again. Anything else propagates.
187
- async #sendMessageWithReconnect(message, { files, maxBytes }) {
435
+ // Retry only when the adapter proves submission never started. Once the Send
436
+ // action has occurred, a dead transport or missing bubble is commit-unknown;
437
+ // replaying the payload could duplicate provider work and attachments.
438
+ async #sendMessageWithReconnect(message, {
439
+ files,
440
+ maxBytes,
441
+ allowFreshReconnect,
442
+ allowAssistantContinuation,
443
+ requireAttachments,
444
+ outboundId,
445
+ }) {
188
446
  try {
189
- await this.adapter.sendMessage(message, { files, maxBytes });
447
+ return await this.adapter.sendMessage(message, {
448
+ files,
449
+ maxBytes,
450
+ outboundId,
451
+ allowAssistantContinuation,
452
+ requireAttachments,
453
+ });
190
454
  } catch (error) {
191
455
  if (isConnectionLostError(error)) {
192
- await this.#reconnectAndRestore();
193
- await this.adapter.sendMessage(message, { files, maxBytes });
194
- return;
195
- }
196
- if (error?.code === "SEND_NOT_DETECTED") {
197
- await this.adapter.sendMessage(message, { files, maxBytes });
198
- return;
456
+ const sendStatus = this.adapter.getLastSendStatus?.() ?? "commit-unknown";
457
+ if (sendStatus !== "not-submitted") {
458
+ throw new BrowserAdapterError(
459
+ `${this.adapter.providerName ?? "Web provider"} connection was lost after submission may have started. `
460
+ + "The message was not sent again automatically.",
461
+ {
462
+ code: "SEND_COMMIT_UNKNOWN",
463
+ recoverable: false,
464
+ cause: error,
465
+ },
466
+ );
467
+ }
468
+ await this.#reconnectAndRestore({
469
+ allowFresh: allowFreshReconnect,
470
+ checkpointIdentity: (restoration) => (
471
+ this.#refreshPendingOutboundIdentity(
472
+ outboundId,
473
+ restoration,
474
+ { required: true },
475
+ )
476
+ ),
477
+ });
478
+ await this.nativeMusicReceiver?.prepare();
479
+ return await this.adapter.sendMessage(message, {
480
+ files,
481
+ maxBytes,
482
+ outboundId,
483
+ allowAssistantContinuation,
484
+ requireAttachments,
485
+ });
199
486
  }
200
487
  throw error;
201
488
  }
202
489
  }
203
490
 
204
- // Reconnects to the still-alive Chrome and navigates back to the session's
205
- // conversation, so the DOM state needed by the adapter is restored.
206
- async #reconnectAndRestore() {
207
- await this.adapter.reconnect?.(this.session.state.conversationUrl);
208
- await this.adapter.startConversation(
209
- this.session.state.conversationUrl,
210
- {
211
- expectedAssistantMessageId: this.session.state.lastAssistantMessageId
212
- ?? null,
213
- },
491
+ // Reconnects to the still-alive Chrome and restores the exact page target when
492
+ // possible. Once a context-dependent message has been sent, landing on a fresh
493
+ // page is never a safe restoration.
494
+ async #reconnectAndRestore({
495
+ allowFresh = false,
496
+ checkpointIdentity = null,
497
+ } = {}) {
498
+ // Navigation callbacks must not persist a replacement root target without
499
+ // the pending record that causally owns it. Suppress them until the caller's
500
+ // checkpoint writes both identities in one Session transition.
501
+ this.deferConversationIdentityPersistence = true;
502
+ try {
503
+ await this.conversationIdentityQueue.catch(() => null);
504
+ await this.adapter.reconnect?.(
505
+ this.session.state.conversationUrl,
506
+ { preferredTargetId: this.session.state.conversationTargetId ?? null },
507
+ );
508
+ const restoration = await this.adapter.startConversation(
509
+ this.session.state.conversationUrl,
510
+ {
511
+ expectedAssistantMessageId: this.session.state.lastAssistantMessageId
512
+ ?? null,
513
+ expectedUserMessageId: this.session.state.lastUserMessageId ?? null,
514
+ },
515
+ );
516
+ if (
517
+ restoration?.status !== "restored-existing"
518
+ && !(allowFresh && restoration?.status === "verified-fresh")
519
+ ) {
520
+ throw new BrowserAdapterError(
521
+ `${this.adapter.providerName} conversation disappeared while a turn was in progress. `
522
+ + "Refusing to continue on a blank page; resume the session to recover safely.",
523
+ {
524
+ code: "CONVERSATION_RESTORE_REQUIRED",
525
+ recoverable: false,
526
+ details: { restorationStatus: restoration?.status ?? null },
527
+ },
528
+ );
529
+ }
530
+ const checkpoint = checkpointIdentity
531
+ ? await checkpointIdentity(restoration)
532
+ : null;
533
+ return { restoration, checkpoint };
534
+ } finally {
535
+ this.deferConversationIdentityPersistence = false;
536
+ }
537
+ }
538
+
539
+ async #freshRebuildBlockers({ instruction = null } = {}) {
540
+ const state = this.session.state;
541
+ const blockers = [];
542
+ if (state.pendingOutbound != null) {
543
+ blockers.push("unconfirmed outbound message");
544
+ }
545
+ if (state.pendingAssistantTurn != null) {
546
+ blockers.push("pending assistant handoff");
547
+ }
548
+ if (state.pendingAuxiliaryTurn != null) {
549
+ blockers.push("pending auxiliary provider turn");
550
+ }
551
+ if (state.pendingToolResult != null) {
552
+ blockers.push("pending tool result");
553
+ }
554
+ if (Object.keys(state.completedTools ?? {}).length > 0) {
555
+ blockers.push("completed tool history");
556
+ }
557
+ if (Object.keys(state.sideEffectTools ?? {}).length > 0) {
558
+ blockers.push("side-effect history");
559
+ }
560
+ if (state.lastMessage != null) {
561
+ blockers.push("completed assistant result");
562
+ }
563
+ if (
564
+ state.lastAssistantMessageId != null
565
+ || await this.session.hasEvent("model.message_complete")
566
+ ) {
567
+ blockers.push("completed or ambiguous assistant response");
568
+ }
569
+
570
+ const suppliedInstruction = instruction?.trim() ?? "";
571
+ const followUps = state.followUps ?? [];
572
+ const persistedInstructions = [
573
+ ...new Set(followUps.map((item) => item?.instruction?.trim()).filter(Boolean)),
574
+ ];
575
+ const followUpsHaveAttachments = followUps.some((item) =>
576
+ item?.attachments != null
577
+ && (!Array.isArray(item.attachments) || item.attachments.length > 0)
214
578
  );
579
+ if (
580
+ followUpsHaveAttachments
581
+ || persistedInstructions.length > 1
582
+ || (
583
+ suppliedInstruction
584
+ && persistedInstructions.length === 1
585
+ && persistedInstructions[0] !== suppliedInstruction
586
+ )
587
+ ) {
588
+ blockers.push("earlier follow-up history");
589
+ }
590
+ const rebuildInstruction = suppliedInstruction
591
+ || (persistedInstructions.length === 1 ? persistedInstructions[0] : null);
592
+
593
+ const transcript = await this.session.readTranscript();
594
+ if (transcript.items.length !== 1) {
595
+ blockers.push(
596
+ transcript.items.length === 0
597
+ ? "missing opening transcript"
598
+ : "canonical transcript history",
599
+ );
600
+ } else {
601
+ const item = transcript.items[0].item;
602
+ const hasNoAttachments = item?.attachments == null
603
+ || (Array.isArray(item.attachments) && item.attachments.length === 0);
604
+ const isOriginalUser = item?.type === "message"
605
+ && item.role === "user"
606
+ && messageText(item) === String(state.task ?? "")
607
+ && hasNoAttachments;
608
+ if (!isOriginalUser) {
609
+ blockers.push("canonical transcript history");
610
+ }
611
+ }
612
+ return { blockers, transcript, rebuildInstruction };
215
613
  }
216
614
 
217
615
  buildToolResultMessage(result, { suffix = "" } = {}) {
218
616
  const limit = this.limits.maxBrowserToolResultBytes;
219
- const nonResultBytes = utf8ByteLength(appendSystemReminder(suffix));
617
+ const nonResultBytes = utf8ByteLength(appendSystemReminder(suffix,
618
+ this.nativeImageReceiver ? NATIVE_IMAGE_SYSTEM_REMINDER : DEFAULT_SYSTEM_REMINDER))
619
+ + OUTBOUND_CORRELATION_RESERVE_BYTES;
220
620
  const resultBudget = limit - nonResultBytes;
221
621
  if (resultBudget < 512) {
222
622
  throw new RangeError(
@@ -226,10 +626,365 @@ export class AgentRuntime {
226
626
  return `${serializeToolResult(result, { maxBytes: resultBudget })}${suffix}`;
227
627
  }
228
628
 
229
- async sendToolResult(result, { suffix = "" } = {}) {
230
- await this.sendMessage(this.buildToolResultMessage(result, { suffix }), {
231
- maxBytes: this.limits.maxBrowserToolResultBytes,
629
+ async sendToolResult(result, {
630
+ suffix = "",
631
+ runtimeTurn = null,
632
+ } = {}) {
633
+ return await this.sendMessage(
634
+ this.buildToolResultMessage(result, { suffix }),
635
+ {
636
+ maxBytes: this.limits.maxBrowserToolResultBytes,
637
+ outboundKind: "tool_result",
638
+ runtimeTurn,
639
+ pendingToolAcknowledgement: {
640
+ callId: result.callId ?? null,
641
+ name: result.name ?? null,
642
+ operationSignature: result.operationSignature ?? null,
643
+ },
644
+ },
645
+ );
646
+ }
647
+
648
+ // Runs a provider-native helper turn (currently used by image.generate) in
649
+ // the same web conversation without exposing that turn as another WTAgent
650
+ // protocol exchange. The send still goes through the adapter's full causal
651
+ // checks, and both provider message identities are checkpointed before the
652
+ // caller is allowed to send the tool result.
653
+ async runAuxiliaryTurn({
654
+ text,
655
+ files = [],
656
+ timeoutMs = this.limits.modelTurnTimeoutMs,
657
+ waitForCompletion,
658
+ }) {
659
+ if (typeof waitForCompletion !== "function") {
660
+ throw new TypeError("An auxiliary turn requires a completion callback.");
661
+ }
662
+ this.#assertProviderReadyForMessage();
663
+ const outboundId = randomUUID();
664
+ const message = appendSystemReminder(
665
+ text,
666
+ `This is an internal WTAgent provider operation. Perform only the requested operation; do not emit the WTAgent XML protocol.${OUTBOUND_CORRELATION_PREFIX}${outboundId}.`,
667
+ );
668
+ const pendingOutbound = {
669
+ kind: "auxiliary_tool_turn",
670
+ outboundId,
671
+ messageHash: createHash("sha256").update(message).digest("hex"),
672
+ preparedAt: new Date().toISOString(),
673
+ transcriptItems: [],
674
+ runtimeTurn: null,
675
+ pendingToolAcknowledgement: null,
676
+ conversationUrl: this.session.state.conversationUrl ?? null,
677
+ conversationTargetId: this.session.state.conversationTargetId ?? null,
678
+ };
679
+ await this.#queueSessionUpdate({ pendingOutbound });
680
+
681
+ let sendResult;
682
+ try {
683
+ sendResult = await this.#sendMessageWithReconnect(message, {
684
+ files,
685
+ maxBytes: null,
686
+ allowFreshReconnect: false,
687
+ allowAssistantContinuation: false,
688
+ requireAttachments: files.length > 0,
689
+ outboundId,
690
+ });
691
+ await this.#queueSessionOperation(() => (
692
+ this.session.commitPendingAuxiliaryTurn({
693
+ outboundId,
694
+ turn: {
695
+ conversationUrl: sendResult?.conversationUrl
696
+ ?? this.session.state.conversationUrl,
697
+ conversationTargetId: sendResult?.conversationTargetId
698
+ ?? this.session.state.conversationTargetId,
699
+ userMessageId: sendResult?.userMessageId ?? null,
700
+ userTurn: sendResult?.userTurn ?? null,
701
+ assistantBaseline: sendResult?.assistantBaseline ?? null,
702
+ },
703
+ })
704
+ ));
705
+ } catch (error) {
706
+ const sendStatus = this.adapter.getLastSendStatus?.() ?? "commit-unknown";
707
+ await this.#queueSessionOperation(async () => {
708
+ if (this.session.state.pendingOutbound?.outboundId === outboundId) {
709
+ await this.session.update({
710
+ pendingOutbound: sendStatus === "not-submitted"
711
+ ? null
712
+ : {
713
+ ...pendingOutbound,
714
+ status: "commit-unknown",
715
+ failedAt: new Date().toISOString(),
716
+ },
717
+ });
718
+ }
719
+ }).catch(() => {});
720
+ if (sendStatus !== "not-submitted") {
721
+ error.completionUnknown = true;
722
+ }
723
+ throw error;
724
+ } finally {
725
+ await this.#syncConversationIdentity({ suppressErrors: true });
726
+ }
727
+
728
+ let value;
729
+ try {
730
+ value = await waitForCompletion({
731
+ adapter: this.adapter,
732
+ page: this.adapter.page,
733
+ sendResult,
734
+ timeoutMs,
735
+ });
736
+ const completion = await this.adapter.captureAuxiliaryTurnCompletion?.({
737
+ timeoutMs: Math.min(timeoutMs, 30_000),
738
+ })
739
+ ?? null;
740
+ const assistantMessageId = completion?.assistantMessageId
741
+ ?? await this.adapter.getLastAssistantMessageId?.()
742
+ ?? null;
743
+ const assistantTurn = completion?.assistantTurn
744
+ ?? await this.adapter.getLastAssistantTurn?.()
745
+ ?? null;
746
+ await this.#queueSessionOperation(() => (
747
+ this.session.completePendingAuxiliaryTurn({
748
+ outboundId,
749
+ assistantMessageId,
750
+ assistantTurn,
751
+ })
752
+ ));
753
+ return value;
754
+ } catch (error) {
755
+ // A provider can finish the visible assistant turn yet fail while WTAgent
756
+ // downloads or validates its artifact. Close that browser-turn checkpoint
757
+ // when its final boundary is still provable; otherwise retain it so a
758
+ // later resume cannot blindly submit a duplicate image request.
759
+ try {
760
+ const completion = await this.adapter.captureAuxiliaryTurnCompletion?.({
761
+ timeoutMs: Math.min(timeoutMs, 5_000),
762
+ });
763
+ if (completion) {
764
+ await this.#queueSessionOperation(() => (
765
+ this.session.completePendingAuxiliaryTurn({
766
+ outboundId,
767
+ assistantMessageId: completion.assistantMessageId ?? null,
768
+ assistantTurn: completion.assistantTurn ?? null,
769
+ })
770
+ ));
771
+ }
772
+ } catch {
773
+ // Keep pendingAuxiliaryTurn intact when completion cannot be proven.
774
+ }
775
+ // If the completion callback produced a temporary artifact but the
776
+ // assistant-boundary checkpoint failed, ownership never reaches the
777
+ // ImageGenerationService, so clean it here.
778
+ await value?.cleanup?.().catch(() => null);
779
+ error.completionUnknown = true;
780
+ throw error;
781
+ } finally {
782
+ await this.#syncConversationIdentity({ suppressErrors: true });
783
+ }
784
+ }
785
+
786
+ async #appendHandoffTranscript(activeHandoff, suffix, item) {
787
+ if (!activeHandoff?.handoffId) {
788
+ throw new Error("Cannot append transcript without an assistant handoff ID.");
789
+ }
790
+ return await this.session.appendTranscriptItemOnce(item, {
791
+ idempotencyKey: `${activeHandoff.handoffId}:${suffix}`,
792
+ });
793
+ }
794
+
795
+ #inferPendingRuntimeTurn(pendingOutbound) {
796
+ if (Number.isSafeInteger(pendingOutbound?.runtimeTurn)) {
797
+ return pendingOutbound.runtimeTurn;
798
+ }
799
+ const currentTurn = Number(this.session.state.turn || 0);
800
+ if ([
801
+ "empty_response_recovery",
802
+ "dead_request_recovery",
803
+ "generation_failed_recovery",
804
+ ].includes(pendingOutbound?.kind)) {
805
+ return Math.max(1, currentTurn);
806
+ }
807
+ return currentTurn + 1;
808
+ }
809
+
810
+ #recoveryHandoffPatch(outcome) {
811
+ return {
812
+ conversationUrl: outcome.conversationUrl,
813
+ conversationTargetId: outcome.conversationTargetId,
814
+ userMessageId: outcome.userMessageId,
815
+ userTurn: outcome.userTurn,
816
+ preOutboundMarkerIds: outcome.preOutboundMarkerIds ?? [],
817
+ assistantBaseline: outcome.assistantBaseline ?? null,
818
+ assistantCandidateMessageId:
819
+ outcome.assistantCandidateMessageId ?? null,
820
+ assistantCandidateTurn: outcome.assistantCandidateTurn ?? null,
821
+ };
822
+ }
823
+
824
+ async #refreshHandoffAfterRestore(handoff, restoration) {
825
+ const conversationUrl = restoration?.conversationUrl
826
+ ?? this.session.state.conversationUrl
827
+ ?? handoff.conversationUrl;
828
+ const conversationTargetId = restoration?.targetId
829
+ ?? this.session.state.conversationTargetId
830
+ ?? handoff.conversationTargetId;
831
+ const refreshed = await this.#queueSessionOperation(() => (
832
+ this.session.refreshPendingAssistantTurn(handoff.handoffId, {
833
+ conversationUrl,
834
+ conversationTargetId,
835
+ })
836
+ ));
837
+ this.adapter.acceptRecoveredOutboundAnchor?.({
838
+ conversationUrl,
839
+ conversationTargetId,
840
+ userMessageId: refreshed.userMessageId,
841
+ userTurn: refreshed.userTurn,
842
+ assistantBaseline: refreshed.assistantBaseline,
843
+ assistantCandidateMessageId:
844
+ refreshed.assistantCandidateMessageId ?? null,
845
+ assistantCandidateTurn: refreshed.assistantCandidateTurn ?? null,
232
846
  });
847
+ return refreshed;
848
+ }
849
+
850
+ async #recoverPendingConversation({ pendingOutbound, pendingAssistantTurn }) {
851
+ const outboundId = pendingOutbound?.outboundId
852
+ ?? pendingAssistantTurn?.sourceOutboundId
853
+ ?? null;
854
+ if (!isValidOutboundCorrelationId(outboundId)) {
855
+ throw new BrowserAdapterError(
856
+ "The pending browser handoff has no valid outbound correlation ID.",
857
+ { code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
858
+ );
859
+ }
860
+ if (!this.adapter.supportsPendingOutboundRecovery?.()) {
861
+ throw new BrowserAdapterError(
862
+ `${this.adapter.providerName ?? "Web provider"} does not support safe pending outbound recovery.`,
863
+ { code: "OUTBOUND_RECOVERY_UNSUPPORTED", recoverable: false },
864
+ );
865
+ }
866
+ const boundTargetId = pendingOutbound?.conversationTargetId
867
+ ?? (!pendingOutbound
868
+ ? pendingAssistantTurn?.conversationTargetId ?? null
869
+ : null);
870
+ const rootTargetId = this.session.state.conversationTargetId ?? null;
871
+ if (
872
+ boundTargetId
873
+ && rootTargetId
874
+ && boundTargetId !== rootTargetId
875
+ ) {
876
+ throw new BrowserAdapterError(
877
+ "The pending browser handoff target conflicts with the Session target. Refusing to choose either target.",
878
+ { code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
879
+ );
880
+ }
881
+ const targetId = boundTargetId ?? rootTargetId;
882
+ if (!targetId) {
883
+ throw new BrowserAdapterError(
884
+ "Pending outbound recovery requires the exact saved browser target.",
885
+ { code: "RECOVERY_TARGET_UNAVAILABLE", recoverable: false },
886
+ );
887
+ }
888
+
889
+ await this.adapter.launchRecoveryTarget(targetId);
890
+ await this.emit("browser.started", {
891
+ profileDir: this.adapter.profileDir,
892
+ recovery: true,
893
+ targetId,
894
+ });
895
+ const priorForReconciliation = pendingOutbound
896
+ && pendingAssistantTurn?.sourceOutboundId !== pendingOutbound.outboundId
897
+ ? null
898
+ : pendingAssistantTurn;
899
+ const outcome = await this.adapter.reconcilePendingOutbound({
900
+ pendingOutbound,
901
+ priorHandoff: priorForReconciliation,
902
+ conversationTargetId: targetId,
903
+ lastUserMessageId: this.session.state.lastUserMessageId ?? null,
904
+ lastAssistantMessageId:
905
+ this.session.state.lastAssistantMessageId ?? null,
906
+ });
907
+ if (!["waiting", "complete"].includes(outcome?.status)) {
908
+ throw new BrowserAdapterError(
909
+ "Pending outbound recovery did not return a durable assistant boundary.",
910
+ { code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
911
+ );
912
+ }
913
+ const outcomeKind = outcome.conversationUrl
914
+ ? this.adapter.classifyConversationUrl?.(outcome.conversationUrl)
915
+ : null;
916
+ if (
917
+ outcome.conversationTargetId !== targetId
918
+ || !["restorable", "provisional"].includes(outcomeKind)
919
+ ) {
920
+ throw new BrowserAdapterError(
921
+ "Pending outbound recovery returned a different browser target or conversation route.",
922
+ { code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
923
+ );
924
+ }
925
+
926
+ let handoff = pendingAssistantTurn;
927
+ if (pendingOutbound) {
928
+ const pendingToolAcknowledgement =
929
+ pendingOutbound.pendingToolAcknowledgement
930
+ ?? (["tool_result", "pending_tool_result"].includes(pendingOutbound.kind)
931
+ && this.session.state.pendingToolResult
932
+ ? {
933
+ callId: this.session.state.pendingToolResult.callId ?? null,
934
+ name: this.session.state.pendingToolResult.name ?? null,
935
+ operationSignature:
936
+ this.session.state.pendingToolResult.operationSignature ?? null,
937
+ }
938
+ : null);
939
+ handoff = await this.#queueSessionOperation(() => (
940
+ this.session.commitPendingOutboundHandoff({
941
+ outboundId: pendingOutbound.outboundId,
942
+ handoff: {
943
+ ...this.#recoveryHandoffPatch(outcome),
944
+ runtimeTurn: this.#inferPendingRuntimeTurn(pendingOutbound),
945
+ pendingToolAcknowledgement,
946
+ },
947
+ })
948
+ ));
949
+ await this.emit("outbound.reconciled", {
950
+ outboundId: pendingOutbound.outboundId,
951
+ kind: pendingOutbound.kind ?? null,
952
+ status: outcome.status,
953
+ userMessageId: outcome.userMessageId ?? null,
954
+ });
955
+ } else if (handoff) {
956
+ handoff = await this.#queueSessionOperation(() => (
957
+ this.session.refreshPendingAssistantTurn(
958
+ handoff.handoffId,
959
+ this.#recoveryHandoffPatch(outcome),
960
+ )
961
+ ));
962
+ }
963
+
964
+ // Recovery reconciles text/identity. A not-yet-checkpointed native turn
965
+ // still needs the ordinary image-aware completion reader before sealing it.
966
+ if (outcome.status === "complete" && ((!this.nativeImageReceiver && !this.nativeMusicReceiver) || handoff.status === "complete")) {
967
+ const outcomeHash = createHash("sha256")
968
+ .update(outcome.rawResponse)
969
+ .digest("hex");
970
+ if (outcome.responseHash && outcome.responseHash !== outcomeHash) {
971
+ throw new BrowserAdapterError(
972
+ "Recovered assistant response hash does not match its raw content.",
973
+ { code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
974
+ );
975
+ }
976
+ handoff = await this.#queueSessionOperation(() => (
977
+ this.session.completePendingAssistantTurn({
978
+ handoffId: handoff.handoffId,
979
+ assistantMessageId: outcome.assistantMessageId ?? null,
980
+ assistantTurn: outcome.assistantTurn ?? null,
981
+ rawResponse: outcome.rawResponse,
982
+ nativeImages: handoff.nativeImages ?? [],
983
+ nativeMusic: handoff.nativeMusic ?? [],
984
+ })
985
+ ));
986
+ }
987
+ return handoff;
233
988
  }
234
989
 
235
990
  async run({
@@ -243,8 +998,65 @@ export class AgentRuntime {
243
998
  projectRoot,
244
999
  } = this.session.state;
245
1000
  const previousConversationUrl = this.session.state.conversationUrl;
246
- await this.session.recoverInterruptedSideEffects();
247
- const pendingToolResult = this.session.state.pendingToolResult;
1001
+ const pendingOutboundAtStart = this.session.state.pendingOutbound;
1002
+ const pendingAssistantAtStart = this.session.state.pendingAssistantTurn;
1003
+ const pendingAuxiliaryAtStart = this.session.state.pendingAuxiliaryTurn;
1004
+ if (pendingAuxiliaryAtStart != null) {
1005
+ throw new BrowserAdapterError(
1006
+ "An image/provider auxiliary turn was interrupted after submission. "
1007
+ + "WTAgent will not replay it automatically because that could create a duplicate remote artifact.",
1008
+ { code: "AUXILIARY_TURN_RECOVERY_REQUIRED", recoverable: false },
1009
+ );
1010
+ }
1011
+ const hasPendingRecovery = pendingOutboundAtStart != null
1012
+ || pendingAssistantAtStart != null;
1013
+ if (this.nativeMusicReceiver && resume && !hasPendingRecovery
1014
+ && !instruction?.trim() && files.length === 0 && this.session.state.lastMusicResult) {
1015
+ // Bare resume of a completed generation must not spend quota again.
1016
+ const result = this.session.state.lastMusicResult;
1017
+ await this.emit("run.completed", result);
1018
+ return { sessionId: this.session.sessionId, ...result };
1019
+ }
1020
+ const supportsStrictRecovery = Boolean(
1021
+ this.adapter.supportsPendingOutboundRecovery?.(),
1022
+ );
1023
+ const strictRecoveryAtStart = pendingOutboundAtStart != null
1024
+ || (pendingAssistantAtStart != null && supportsStrictRecovery);
1025
+ const legacyAssistantRecoveryAtStart = pendingOutboundAtStart == null
1026
+ && pendingAssistantAtStart != null
1027
+ && !supportsStrictRecovery;
1028
+ if (
1029
+ hasPendingRecovery
1030
+ && (
1031
+ !resume
1032
+ || Boolean(instruction?.trim())
1033
+ || (files ?? []).length > 0
1034
+ )
1035
+ ) {
1036
+ throw new BrowserAdapterError(
1037
+ "A pending browser handoff can only be recovered with a bare resume. "
1038
+ + "Do not add an instruction or attachment until recovery completes.",
1039
+ { code: "RECOVERY_REQUIRES_BARE_RESUME", recoverable: false },
1040
+ );
1041
+ }
1042
+ const attachments = (files ?? []).map((file) => ({
1043
+ name: file.name ?? null,
1044
+ path: file.path ?? null,
1045
+ }));
1046
+ const messageOptions = attachments.length > 0 ? { attachments } : {};
1047
+
1048
+ // Persist the opening user intent (including attachment metadata) before any
1049
+ // browser work. A later recovery can then distinguish a replayable untouched
1050
+ // task from an unknowable empty legacy rollout.
1051
+ if (!resume) {
1052
+ const transcript = await this.session.readTranscript();
1053
+ if (transcript.items.length === 0) {
1054
+ await this.session.appendTranscriptItemOnce(
1055
+ userMessage(task, messageOptions),
1056
+ { idempotencyKey: `session:${this.session.sessionId}:opening` },
1057
+ );
1058
+ }
1059
+ }
248
1060
 
249
1061
  await this.session.update({
250
1062
  phase: "initializing",
@@ -253,10 +1065,49 @@ export class AgentRuntime {
253
1065
  });
254
1066
  await this.emit("runtime.initializing");
255
1067
 
1068
+ const modelTurnTimeoutMs = this.limits.modelTurnTimeoutMs;
1069
+ let pendingToolResult;
1070
+ let awaitingPendingAcknowledgement = false;
1071
+ let replayGuard = null;
1072
+ let protocolErrors = 0;
1073
+ const baseTurn = resume ? Number(this.session.state.turn || 0) : 0;
1074
+
1075
+ if (strictRecoveryAtStart) {
1076
+ const recoveredHandoff = await this.#recoverPendingConversation({
1077
+ pendingOutbound: pendingOutboundAtStart,
1078
+ pendingAssistantTurn: pendingAssistantAtStart,
1079
+ });
1080
+ // Recovery proves the browser boundary before touching a possibly-running
1081
+ // side-effect ledger. The existing local exactly-once rules then apply.
1082
+ await this.session.recoverInterruptedSideEffects();
1083
+ pendingToolResult = this.session.state.pendingToolResult;
1084
+ awaitingPendingAcknowledgement = Boolean(
1085
+ recoveredHandoff.pendingToolAcknowledgement,
1086
+ );
1087
+ replayGuard = pendingToolResult?.operationSignature
1088
+ ? {
1089
+ signature: pendingToolResult.operationSignature,
1090
+ result: pendingToolResult,
1091
+ }
1092
+ : null;
1093
+ await this.session.update({
1094
+ phase: "running",
1095
+ activeMode: null,
1096
+ });
1097
+ await this.emit("conversation.started", {
1098
+ url: recoveredHandoff.conversationUrl,
1099
+ recovered: true,
1100
+ });
1101
+ } else {
256
1102
  // On resume, prefer an existing tab already showing the conversation so
257
1103
  // repeated resumes reuse the same tab instead of piling up new ones.
258
1104
  await this.adapter.launch(
259
1105
  resume ? this.session.state.conversationUrl : null,
1106
+ {
1107
+ preferredTargetId: resume
1108
+ ? this.session.state.conversationTargetId ?? null
1109
+ : null,
1110
+ },
260
1111
  );
261
1112
  await this.emit("browser.started", {
262
1113
  profileDir: this.adapter.profileDir,
@@ -288,47 +1139,156 @@ export class AgentRuntime {
288
1139
  await this.emit("browser.authenticated");
289
1140
  }
290
1141
 
291
- await this.adapter.startConversation(
1142
+ let restoration = await this.adapter.startConversation(
292
1143
  resume ? previousConversationUrl : null,
293
1144
  {
294
1145
  expectedAssistantMessageId: resume
295
1146
  ? this.session.state.lastAssistantMessageId
296
1147
  : null,
1148
+ expectedUserMessageId: resume
1149
+ ? this.session.state.lastUserMessageId
1150
+ : null,
297
1151
  },
298
1152
  );
1153
+ if (
1154
+ restoration?.status !== "restored-existing"
1155
+ && restoration?.status !== "verified-fresh"
1156
+ ) {
1157
+ throw new BrowserAdapterError(
1158
+ `${this.adapter.providerName} did not report whether conversation history was restored.`,
1159
+ {
1160
+ code: "CONVERSATION_RESTORE_UNVERIFIED",
1161
+ recoverable: false,
1162
+ },
1163
+ );
1164
+ }
1165
+ if (
1166
+ legacyAssistantRecoveryAtStart
1167
+ && restoration.status !== "restored-existing"
1168
+ ) {
1169
+ throw new BrowserAdapterError(
1170
+ `${this.adapter.providerName} could not restore the confirmed pending assistant turn. `
1171
+ + "Refusing to continue it on a blank conversation.",
1172
+ {
1173
+ code: "CONVERSATION_RESTORE_REQUIRED",
1174
+ recoverable: false,
1175
+ },
1176
+ );
1177
+ }
1178
+ await this.#syncConversationIdentity({
1179
+ identity: {
1180
+ conversationUrl: restoration.conversationUrl,
1181
+ targetId: restoration.targetId ?? null,
1182
+ kind: restoration.status === "verified-fresh"
1183
+ ? "fresh"
1184
+ : this.adapter.classifyConversationUrl?.(restoration.conversationUrl),
1185
+ },
1186
+ allowFresh: restoration.status === "verified-fresh",
1187
+ });
1188
+
1189
+ const resumedExistingConversation = resume
1190
+ && restoration.status === "restored-existing";
1191
+ const rebuildingFreshConversation = resume
1192
+ && restoration.status === "verified-fresh";
1193
+ let effectiveInstruction = instruction;
1194
+ if (rebuildingFreshConversation) {
1195
+ const {
1196
+ blockers,
1197
+ rebuildInstruction,
1198
+ } = await this.#freshRebuildBlockers({ instruction });
1199
+ if (blockers.length > 0) {
1200
+ throw new BrowserAdapterError(
1201
+ `${this.adapter.providerName} opened an empty conversation, but this session has `
1202
+ + `history that cannot be replayed safely (${blockers.join(", ")}). `
1203
+ + "The browser message was not sent.",
1204
+ {
1205
+ code: "CONVERSATION_REBUILD_UNSAFE",
1206
+ recoverable: false,
1207
+ details: { blockers },
1208
+ },
1209
+ );
1210
+ }
1211
+ effectiveInstruction = rebuildInstruction;
1212
+ await this.emit("conversation.rebuilding_fresh", {
1213
+ previousUrl: previousConversationUrl,
1214
+ url: restoration.conversationUrl,
1215
+ });
1216
+ }
1217
+
1218
+ // Only normalize an interrupted side-effect ledger after the browser
1219
+ // restoration decision. An unsafe fresh-page fallback must leave every
1220
+ // pending/result/side-effect record untouched for deliberate inspection.
1221
+ await this.session.recoverInterruptedSideEffects();
1222
+ pendingToolResult = this.session.state.pendingToolResult;
299
1223
 
300
1224
  // Optional setup after authentication and the target conversation are
301
1225
  // ready, but before the first message is sent. Interactive CLI runs use
302
1226
  // this to let the user choose a model directly on the provider website.
303
1227
  // WTAgent never names, selects, or overrides a provider-specific model.
304
- // Resumes skip this setup so an existing conversation is not interrupted.
305
- if (!resume && this.postAuthSetup) {
1228
+ // Existing conversations keep their model context. A verified fresh rebuild
1229
+ // is a genuinely new web chat, so interactive runs offer the same browser-side
1230
+ // model choice as an ordinary bootstrap.
1231
+ if (restoration.status === "verified-fresh" && this.postAuthSetup) {
306
1232
  await this.postAuthSetup({ adapter: this.adapter });
1233
+ // Browser-side setup is user-controlled and can navigate the tab. Reopen
1234
+ // and re-verify an empty chat after it returns so the bootstrap can never
1235
+ // land in a history conversation selected during setup.
1236
+ restoration = await this.adapter.startConversation();
1237
+ if (restoration?.status !== "verified-fresh") {
1238
+ throw new BrowserAdapterError(
1239
+ `${this.adapter.providerName} did not remain on a verified empty conversation after browser setup.`,
1240
+ {
1241
+ code: "CONVERSATION_NOT_FRESH",
1242
+ recoverable: false,
1243
+ },
1244
+ );
1245
+ }
1246
+ await this.#syncConversationIdentity({
1247
+ identity: {
1248
+ conversationUrl: restoration.conversationUrl,
1249
+ targetId: restoration.targetId ?? null,
1250
+ kind: "fresh",
1251
+ },
1252
+ allowFresh: true,
1253
+ });
307
1254
  }
308
1255
 
309
- const modelTurnTimeoutMs = this.limits.modelTurnTimeoutMs;
310
-
311
1256
  await this.session.update({
312
1257
  phase: "running",
313
- conversationUrl: await this.adapter.getConversationUrl(),
314
1258
  activeMode: null,
315
1259
  });
316
1260
  await this.emit("conversation.started", {
317
1261
  url: this.session.state.conversationUrl,
318
1262
  });
319
1263
 
1264
+ if (legacyAssistantRecoveryAtStart) {
1265
+ const recoveredHandoff = await this.#refreshHandoffAfterRestore(
1266
+ pendingAssistantAtStart,
1267
+ restoration,
1268
+ );
1269
+ awaitingPendingAcknowledgement = Boolean(
1270
+ recoveredHandoff.pendingToolAcknowledgement,
1271
+ );
1272
+ replayGuard = pendingToolResult?.operationSignature
1273
+ ? {
1274
+ signature: pendingToolResult.operationSignature,
1275
+ result: pendingToolResult,
1276
+ }
1277
+ : null;
1278
+ }
320
1279
  let initialMessage;
321
1280
  let initialKind;
322
1281
  // The web transport gets `.web` (XML/marked text). The portable rollout
323
- // records only the real user message; WTAgent scaffolding stays transport-only.
1282
+ // records only confirmed user messages; WTAgent scaffolding stays transport-only.
324
1283
  let initialTranscript = [];
325
- // @file attachments (if any) accompany the first user message of this run.
326
- const attachments = (files ?? []).map((file) => ({
327
- name: file.name ?? null,
328
- path: file.path ?? null,
329
- }));
330
- const messageOptions = attachments.length > 0 ? { attachments } : {};
331
- if (resume && pendingToolResult && !inPlaceRecovery) {
1284
+ if (!legacyAssistantRecoveryAtStart) {
1285
+ if (this.nativeMusicReceiver) {
1286
+ if (inPlaceRecovery) throw new Error("Music generation is not automatically repeated; resume the pending turn without a new instruction.");
1287
+ const request = effectiveInstruction?.trim() || task;
1288
+ initialMessage = `Create the following music using the native Music tool.\n\n${request}`;
1289
+ initialTranscript = resume && effectiveInstruction?.trim() ? [userMessage(effectiveInstruction.trim(), messageOptions)] : [];
1290
+ initialKind = resume ? "music_follow_up" : "music_generation";
1291
+ } else if (resumedExistingConversation && pendingToolResult && !inPlaceRecovery) {
332
1292
  let suffix = "";
333
1293
  if (instruction?.trim()) {
334
1294
  suffix = `\n<resume_instruction>${cdata(instruction)}</resume_instruction>`;
@@ -336,7 +1296,7 @@ export class AgentRuntime {
336
1296
  }
337
1297
  initialMessage = this.buildToolResultMessage(pendingToolResult, { suffix });
338
1298
  initialKind = "pending_tool_result";
339
- } else if (resume && instruction?.trim()) {
1299
+ } else if (resumedExistingConversation && instruction?.trim()) {
340
1300
  // The live web conversation already contains the bootstrap protocol
341
1301
  // and tool catalog. A normal follow-up should be the user's message, not
342
1302
  // another several-thousand-character protocol bootstrap. sendMessage()
@@ -344,7 +1304,7 @@ export class AgentRuntime {
344
1304
  initialMessage = instruction.trim();
345
1305
  initialTranscript = [userMessage(instruction.trim(), messageOptions)];
346
1306
  initialKind = "follow_up";
347
- } else if (resume && inPlaceRecovery) {
1307
+ } else if (resumedExistingConversation && inPlaceRecovery) {
348
1308
  // The original request/tool result is already visible in this live web
349
1309
  // conversation. Ask the provider to continue without duplicating transport
350
1310
  // payloads, attachments, or canonical transcript entries.
@@ -352,67 +1312,127 @@ export class AgentRuntime {
352
1312
  initialKind = "empty_response_recovery";
353
1313
  } else if (resume) {
354
1314
  const prompt = buildResumePrompt({
355
- instruction,
1315
+ instruction: rebuildingFreshConversation
1316
+ ? effectiveInstruction
1317
+ : instruction,
356
1318
  state: this.session.state,
357
1319
  tools: this.registry.list(),
1320
+ nativeImages: Boolean(this.nativeImageReceiver),
358
1321
  });
359
1322
  initialMessage = prompt.web;
360
- initialTranscript = [userMessage(prompt.user, messageOptions)];
361
- initialKind = "resume";
1323
+ // The original task is already the one safe canonical item required by the
1324
+ // rebuild gate. Record only a newly confirmed follow-up instruction.
1325
+ initialTranscript = rebuildingFreshConversation
1326
+ ? (effectiveInstruction?.trim()
1327
+ ? [userMessage(effectiveInstruction.trim(), messageOptions)]
1328
+ : [])
1329
+ : [userMessage(prompt.user, messageOptions)];
1330
+ initialKind = rebuildingFreshConversation ? "fresh_rebuild" : "resume";
362
1331
  } else {
363
1332
  const prompt = buildBootstrapPrompt({
364
1333
  task,
365
1334
  projectRoot,
366
1335
  tools: this.registry.list(),
1336
+ nativeImages: Boolean(this.nativeImageReceiver),
367
1337
  });
368
1338
  initialMessage = prompt.web;
369
- initialTranscript = [userMessage(prompt.user, messageOptions)];
1339
+ // The opening item was persisted before browser setup above.
1340
+ initialTranscript = [];
370
1341
  initialKind = "bootstrap";
371
1342
  }
372
-
373
- for (const item of initialTranscript) {
374
- await this.session.appendTranscriptItem(item);
375
1343
  }
376
- await this.sendMessage(initialMessage, {
377
- files,
378
- maxBytes: initialKind === "pending_tool_result"
379
- ? this.limits.maxBrowserToolResultBytes
380
- : null,
381
- });
382
- let awaitingPendingAcknowledgement = Boolean(pendingToolResult);
383
- let replayGuard = pendingToolResult?.operationSignature
384
- ? {
385
- signature: pendingToolResult.operationSignature,
386
- result: pendingToolResult,
387
- }
388
- : null;
389
- await this.emit("model.message_sent", { kind: initialKind });
390
1344
 
391
- let protocolErrors = 0;
392
- const baseTurn = resume ? Number(this.session.state.turn || 0) : 0;
1345
+ if (!legacyAssistantRecoveryAtStart) {
1346
+ await this.sendMessage(initialMessage, {
1347
+ files,
1348
+ maxBytes: initialKind === "pending_tool_result"
1349
+ ? this.limits.maxBrowserToolResultBytes
1350
+ : null,
1351
+ // A full bootstrap/rebuild prompt may reconnect only when submission was
1352
+ // proven not to have started. Context-dependent messages never opt in.
1353
+ allowFreshReconnect: restoration.status === "verified-fresh",
1354
+ allowAssistantContinuation: initialKind === "empty_response_recovery",
1355
+ outboundKind: initialKind,
1356
+ transcriptItems: initialTranscript,
1357
+ runtimeTurn: baseTurn + 1,
1358
+ pendingToolAcknowledgement: pendingToolResult
1359
+ ? {
1360
+ callId: pendingToolResult.callId ?? null,
1361
+ name: pendingToolResult.name ?? null,
1362
+ operationSignature: pendingToolResult.operationSignature ?? null,
1363
+ }
1364
+ : null,
1365
+ });
1366
+ awaitingPendingAcknowledgement = Boolean(pendingToolResult);
1367
+ replayGuard = pendingToolResult?.operationSignature
1368
+ ? {
1369
+ signature: pendingToolResult.operationSignature,
1370
+ result: pendingToolResult,
1371
+ }
1372
+ : null;
1373
+ await this.emit("model.message_sent", { kind: initialKind });
1374
+ }
1375
+ }
393
1376
 
394
1377
  for (let step = 1; ; step += 1) {
395
- const turnNumber = baseTurn + step;
1378
+ let activeHandoff = this.session.state.pendingAssistantTurn;
1379
+ if (!activeHandoff) {
1380
+ throw new BrowserAdapterError(
1381
+ "The durable assistant handoff disappeared before response processing.",
1382
+ { code: "ASSISTANT_HANDOFF_MISSING", recoverable: false },
1383
+ );
1384
+ }
1385
+ const turnNumber = Number.isSafeInteger(activeHandoff.runtimeTurn)
1386
+ ? activeHandoff.runtimeTurn
1387
+ : baseTurn + step;
396
1388
  await this.session.update({ turn: turnNumber, phase: "waiting_model" });
397
1389
  let raw;
1390
+ let nativeImages = [];
1391
+ let nativeMusic = [];
398
1392
  let emptyAssistantRetries = 0;
399
1393
  let connectionRetries = 0;
400
1394
  for (;;) {
401
1395
  try {
402
- raw = await this.adapter.waitForTurnComplete({
403
- timeoutMs: modelTurnTimeoutMs,
404
- stableWindowMs: this.limits.modelStableWindowMs,
405
- emptyResponseWindowMs: this.limits.emptyAssistantWindowMs,
406
- deadRequestGraceMs: this.limits.deadRequestGraceMs,
407
- onDelta: async (delta) => {
408
- await this.onEvent?.({
409
- type: "model.streaming",
410
- sessionId: this.session.sessionId,
411
- timestamp: new Date().toISOString(),
412
- payload: { delta },
1396
+ if (activeHandoff.status === "complete") {
1397
+ raw = activeHandoff.rawResponse;
1398
+ nativeImages = activeHandoff.nativeImages ?? [];
1399
+ nativeMusic = activeHandoff.nativeMusic ?? [];
1400
+ } else {
1401
+ try {
1402
+ raw = await this.adapter.waitForTurnComplete({
1403
+ timeoutMs: modelTurnTimeoutMs,
1404
+ stableWindowMs: this.limits.modelStableWindowMs,
1405
+ emptyResponseWindowMs: this.limits.emptyAssistantWindowMs,
1406
+ deadRequestGraceMs: this.limits.deadRequestGraceMs,
1407
+ readNativeImages: this.nativeImageReceiver
1408
+ ? (message) => this.nativeImageReceiver.read(message) : null,
1409
+ readNativeMusic: this.nativeMusicReceiver
1410
+ ? (message) => this.nativeMusicReceiver.read(message) : null,
1411
+ onDelta: async (delta) => {
1412
+ await this.onEvent?.({
1413
+ type: "model.streaming",
1414
+ sessionId: this.session.sessionId,
1415
+ timestamp: new Date().toISOString(),
1416
+ payload: { delta },
1417
+ });
1418
+ },
413
1419
  });
414
- },
415
- });
1420
+ if (raw && typeof raw === "object" && Array.isArray(raw.nativeImages)) {
1421
+ nativeImages = raw.nativeImages;
1422
+ raw = raw.text ?? "";
1423
+ } else if (raw && typeof raw === "object" && Array.isArray(raw.nativeMusic)) {
1424
+ nativeMusic = raw.nativeMusic;
1425
+ raw = raw.text ?? "";
1426
+ }
1427
+ } finally {
1428
+ // Navigation events normally persist canonicalization immediately;
1429
+ // this validated fallback runs on every outcome but never masks a
1430
+ // completed reply or the original browser/model error.
1431
+ await this.#syncConversationIdentity({
1432
+ suppressErrors: true,
1433
+ });
1434
+ }
1435
+ }
416
1436
  break;
417
1437
  } catch (error) {
418
1438
  if (isConnectionLostError(error)) {
@@ -420,9 +1440,25 @@ export class AgentRuntime {
420
1440
  throw error;
421
1441
  }
422
1442
  connectionRetries += 1;
423
- await this.#reconnectAndRestore();
1443
+ if (supportsStrictRecovery) {
1444
+ await this.adapter.detach?.();
1445
+ activeHandoff = await this.#recoverPendingConversation({
1446
+ pendingOutbound: null,
1447
+ pendingAssistantTurn: activeHandoff,
1448
+ });
1449
+ } else {
1450
+ const { checkpoint } = await this.#reconnectAndRestore({
1451
+ checkpointIdentity: (restoration) => (
1452
+ this.#refreshHandoffAfterRestore(
1453
+ activeHandoff,
1454
+ restoration,
1455
+ )
1456
+ ),
1457
+ });
1458
+ activeHandoff = checkpoint;
1459
+ }
424
1460
  // The message was already sent before the connection died; resume
425
- // waiting for the reply on the restored page.
1461
+ // waiting for the reply on the restored exact conversation boundary.
426
1462
  continue;
427
1463
  }
428
1464
 
@@ -437,6 +1473,7 @@ export class AgentRuntime {
437
1473
 
438
1474
  const deadRequest = error?.code === "DEAD_ASSISTANT_REQUEST";
439
1475
  const generationFailed = error?.code === "GENERATION_FAILED";
1476
+ if (this.nativeMusicReceiver) throw error;
440
1477
  if (
441
1478
  error?.code !== "EMPTY_ASSISTANT_RESPONSE"
442
1479
  && !deadRequest
@@ -448,7 +1485,6 @@ export class AgentRuntime {
448
1485
  const emptyAssistantMessageId = await this.adapter
449
1486
  .getLastAssistantMessageId?.() ?? null;
450
1487
  await this.session.update({
451
- conversationUrl: await this.adapter.getConversationUrl(),
452
1488
  lastAssistantMessageId: emptyAssistantMessageId
453
1489
  ?? this.session.state.lastAssistantMessageId,
454
1490
  });
@@ -488,13 +1524,25 @@ export class AgentRuntime {
488
1524
  // Do not resend the original request or tool result: both are already
489
1525
  // present in ChatGPT's conversation. This transport-only continuation
490
1526
  // also cannot re-execute a local tool by itself.
491
- await this.sendMessage(
1527
+ const continuationSend = await this.sendMessage(
492
1528
  deadRequest
493
1529
  ? DEAD_REQUEST_CONTINUE_MESSAGE
494
1530
  : generationFailed
495
1531
  ? GENERATION_FAILED_CONTINUE_MESSAGE
496
1532
  : EMPTY_ASSISTANT_CONTINUE_MESSAGE,
1533
+ {
1534
+ allowAssistantContinuation: true,
1535
+ outboundKind: deadRequest
1536
+ ? "dead_request_recovery"
1537
+ : generationFailed
1538
+ ? "generation_failed_recovery"
1539
+ : "empty_response_recovery",
1540
+ runtimeTurn: turnNumber,
1541
+ pendingToolAcknowledgement:
1542
+ activeHandoff.pendingToolAcknowledgement ?? null,
1543
+ },
497
1544
  );
1545
+ activeHandoff = continuationSend.pendingAssistantTurn;
498
1546
  await this.emit("model.message_sent", {
499
1547
  kind: deadRequest
500
1548
  ? "dead_request_recovery"
@@ -505,27 +1553,131 @@ export class AgentRuntime {
505
1553
  });
506
1554
  }
507
1555
  }
508
- const assistantMessageId = await this.adapter
509
- .getLastAssistantMessageId?.() ?? null;
1556
+ const assistantMessageId = activeHandoff.status === "complete"
1557
+ ? activeHandoff.assistantMessageId ?? null
1558
+ : await this.adapter.getLastAssistantMessageId?.() ?? null;
1559
+ const assistantTurn = activeHandoff.status === "complete"
1560
+ ? activeHandoff.assistantTurn ?? null
1561
+ : await this.adapter.getLastAssistantTurn?.() ?? null;
1562
+ if (activeHandoff.status === "waiting") {
1563
+ activeHandoff = await this.#queueSessionOperation(() => (
1564
+ this.session.completePendingAssistantTurn({
1565
+ handoffId: activeHandoff.handoffId,
1566
+ assistantMessageId,
1567
+ assistantTurn,
1568
+ rawResponse: raw,
1569
+ nativeImages,
1570
+ nativeMusic,
1571
+ })
1572
+ ));
1573
+ } else {
1574
+ const rawHash = createHash("sha256").update(raw).digest("hex");
1575
+ if (rawHash !== activeHandoff.responseHash) {
1576
+ throw new BrowserAdapterError(
1577
+ "The durable assistant response changed before processing.",
1578
+ { code: "ASSISTANT_HANDOFF_COLLISION", recoverable: false },
1579
+ );
1580
+ }
1581
+ }
510
1582
  if (awaitingPendingAcknowledgement) {
511
- await this.session.clearPendingToolResult();
512
- awaitingPendingAcknowledgement = false;
1583
+ await this.#queueSessionUpdate({ pendingToolResult: null });
513
1584
  }
514
- await this.session.update({
515
- conversationUrl: await this.adapter.getConversationUrl(),
516
- // Null is meaningful: retaining an older ID would falsely prove only
517
- // that stale history had hydrated on the next resume.
518
- lastAssistantMessageId: assistantMessageId,
519
- });
1585
+ awaitingPendingAcknowledgement = false;
520
1586
  await this.emit("model.message_complete", {
521
1587
  turn: turnNumber,
522
1588
  raw,
523
1589
  assistantMessageId,
524
1590
  });
525
1591
 
1592
+ if (this.nativeMusicReceiver) {
1593
+ let artifacts = activeHandoff.nativeArtifacts;
1594
+ if (!artifacts) {
1595
+ artifacts = nativeMusic.length ? await this.nativeMusicReceiver.save(nativeMusic, {
1596
+ projectRoot, handoffId: activeHandoff.handoffId, assistantMessageId,
1597
+ }) : [];
1598
+ activeHandoff = await this.#queueSessionOperation(() => (
1599
+ this.session.refreshPendingAssistantTurn(activeHandoff.handoffId, { nativeArtifacts: artifacts })
1600
+ ));
1601
+ }
1602
+ const message = [
1603
+ raw,
1604
+ ...artifacts.map((artifact) => `Saved music: ${artifact.localPath} (${artifact.durationSeconds.toFixed(2)} s)`),
1605
+ ...artifacts.map((artifact) => artifact.audioExtractionNote).filter(Boolean),
1606
+ ...(artifacts.length ? [] : ["No music file was generated or saved."]),
1607
+ ].filter(Boolean).join("\n");
1608
+ await this.#appendHandoffTranscript(activeHandoff, "assistant", assistantMessage(message));
1609
+ const result = { message, artifacts };
1610
+ await this.#queueSessionOperation(() => this.session.clearPendingAssistantTurn(activeHandoff.handoffId, {
1611
+ phase: "idle", lastMessage: message, pendingToolResult: null, lastMusicResult: result,
1612
+ }));
1613
+ if (artifacts.length) await this.emit("model.native_music", { turn: turnNumber, artifacts });
1614
+ await this.emit("run.completed", result);
1615
+ return { sessionId: this.session.sessionId, ...result };
1616
+ }
1617
+
1618
+ if (nativeImages.length > 0) {
1619
+ if (!this.nativeImageReceiver) {
1620
+ throw new Error("This session has pending native images but no image receiver.");
1621
+ }
1622
+ let artifacts = activeHandoff.nativeArtifacts;
1623
+ if (!artifacts) {
1624
+ artifacts = await this.nativeImageReceiver.save(nativeImages, {
1625
+ projectRoot,
1626
+ handoffId: activeHandoff.handoffId,
1627
+ assistantMessageId,
1628
+ });
1629
+ activeHandoff = await this.#queueSessionOperation(() => (
1630
+ this.session.refreshPendingAssistantTurn(activeHandoff.handoffId, { nativeArtifacts: artifacts })
1631
+ ));
1632
+ }
1633
+ protocolErrors = 0;
1634
+ const message = [raw, ...artifacts.map((artifact) => `Saved image: ${artifact.localPath}`)]
1635
+ .filter(Boolean).join("\n");
1636
+ await this.#appendHandoffTranscript(activeHandoff, "assistant", assistantMessage(message));
1637
+ await this.emit("model.native_images", { turn: turnNumber, artifacts });
1638
+ await this.emit("model.progress", { turn: turnNumber, message });
1639
+ const receipt = artifacts.map(({ localPath, mimeType, width, height, size, sha256 }) => (
1640
+ { localPath, mimeType, width, height, size, sha256 }
1641
+ ));
1642
+ await this.sendMessage(
1643
+ `<native_image_result>${cdata(JSON.stringify(receipt))}</native_image_result>\n`
1644
+ + "The images from your preceding reply are saved at these verified local paths. "
1645
+ + "Continue the user's original task. Generate further images directly if needed; "
1646
+ + "use local tools for file operations, or finish with a done=true XML answer.",
1647
+ { outboundKind: "native_image_result", runtimeTurn: turnNumber + 1 },
1648
+ );
1649
+ continue;
1650
+ }
1651
+
526
1652
  let parsed;
1653
+ let finalMessage;
1654
+ let usedTrailingProse = false;
527
1655
  try {
528
1656
  parsed = parseAgentResponse(raw);
1657
+ finalMessage = parsed.message;
1658
+ // Some providers put the deliverable after a done=true envelope. Merge
1659
+ // that display text before validating the final answer.
1660
+ if (parsed.done) {
1661
+ const trailing = extractTrailingProse(raw);
1662
+ if (trailing) {
1663
+ finalMessage = [parsed.message.trim(), trailing]
1664
+ .filter(Boolean)
1665
+ .join("\n\n");
1666
+ usedTrailingProse = true;
1667
+ }
1668
+ if (!finalMessage.trim()) {
1669
+ throw new ProtocolError("done=true requires a non-empty final message.");
1670
+ }
1671
+ if (
1672
+ !parsed.toolCall
1673
+ && /<tool_calls[\s>]|<tool_call[\s>]|<invoke[\s>]/i.test(finalMessage)
1674
+ ) {
1675
+ throw new ProtocolError(
1676
+ "Tool calls must use the <tool_call> element, not the <message> text.",
1677
+ );
1678
+ }
1679
+ }
1680
+ // Syntax and semantic errors share one consecutive-retry budget.
529
1681
  protocolErrors = 0;
530
1682
  } catch (error) {
531
1683
  if (!(error instanceof ProtocolError)) {
@@ -546,41 +1698,33 @@ export class AgentRuntime {
546
1698
  );
547
1699
  }
548
1700
 
549
- // The model answered in plain prose without any <agent_response> at
550
- // all. That cannot be a broken tool request (tools only exist inside a
551
- // parsed envelope), so it is safe to treat the prose as the final
552
- // answer: end the run and show it, instead of burning retries on a
553
- // model that deliberately finished the conversation. A reply that DOES
554
- // contain <agent_response but fails to parse keeps the retry path —
555
- // the model tried the protocol and we must not guess its intent.
556
- // A bare <tool_calls>/<invoke> reply (no envelope) is likewise a tool
557
- // REQUEST, never prose: it goes to the protocol-error retry below.
558
- const looksLikeToolRequest = /<tool_calls[\s>]|<tool_call[\s>]|<invoke[\s>]/i
559
- .test(raw);
560
- const plainAnswer = !raw.includes("<agent_response")
561
- && !looksLikeToolRequest
562
- ? stripUiNoiseLines(raw)
563
- : "";
564
- if (plainAnswer) {
565
- await this.emit("protocol.plain_answer", {
566
- snippet: plainAnswer.slice(0, 200),
567
- });
568
- await this.session.update({
569
- phase: "idle",
570
- lastMessage: plainAnswer,
571
- pendingToolResult: null,
572
- });
573
- await this.session.appendTranscriptItem(assistantMessage(plainAnswer));
574
- await this.emit("run.completed", {
575
- message: plainAnswer,
576
- plainAnswer: true,
577
- });
578
- return {
579
- sessionId: this.session.sessionId,
580
- message: plainAnswer,
581
- };
1701
+ // A finished native refusal/clarification is data for the agent, not a
1702
+ // successful image or an empty-response retry. Keep a bounded budget;
1703
+ // malformed XML still follows strict protocol correction below.
1704
+ if (this.nativeImageReceiver && raw.trim() && !/<\/?(?:agent_response|tool_call|tool_calls|invoke|done|message)(?:\s|>)/i.test(raw)) {
1705
+ protocolErrors += 1;
1706
+ await this.#appendHandoffTranscript(activeHandoff, "assistant", assistantMessage(raw));
1707
+ await this.emit("model.progress", { turn: turnNumber, message: raw });
1708
+ if (protocolErrors >= this.limits.maxProtocolErrors) {
1709
+ throw new ProtocolError("The provider repeatedly returned text without a completed XML answer or an image.");
1710
+ }
1711
+ await this.sendMessage(
1712
+ `<native_image_result type="text">${cdata(raw)}</native_image_result>\n`
1713
+ + "Your preceding reply contained only this text; no image was received or saved. "
1714
+ + "Continue the original task based on this result. If generation is unavailable or declined, "
1715
+ + "explain it or ask the user a specific question in a done=true XML answer. Do not automatically retry the same image request.",
1716
+ { outboundKind: "native_image_text", runtimeTurn: turnNumber + 1 },
1717
+ );
1718
+ continue;
582
1719
  }
583
1720
 
1721
+ // Absence of an envelope is a protocol failure, not proof of done.
1722
+ // This also covers UI-only text and partial native-artifact responses.
1723
+ await this.adapter.writeDiagnostics?.("protocol-invalid", {
1724
+ turn: turnNumber,
1725
+ assistantMessageId,
1726
+ error: error.message,
1727
+ }).catch(() => null);
584
1728
  protocolErrors += 1;
585
1729
  await this.emit("protocol.invalid", {
586
1730
  message: error.message,
@@ -591,79 +1735,39 @@ export class AgentRuntime {
591
1735
  `Protocol failed ${protocolErrors} consecutive times: ${error.message}`,
592
1736
  );
593
1737
  }
594
- await this.sendMessage(serializeProtocolError(error));
1738
+ await this.sendMessage(serializeProtocolError(error), {
1739
+ allowAssistantContinuation: true,
1740
+ outboundKind: "protocol_correction",
1741
+ runtimeTurn: turnNumber + 1,
1742
+ });
595
1743
  continue;
596
1744
  }
597
1745
 
598
- // When the run finishes, some models (GLM especially) put their real
599
- // deliverable AFTER the envelope: a short done/true stub followed by the
600
- // full markdown/HTML answer. That trailing prose is pure display content
601
- // (it cannot trigger a tool), so merge it into the final message instead
602
- // of dropping it. Non-done turns keep the protocol message untouched.
603
- let finalMessage = parsed.message;
604
- let usedTrailingProse = false;
605
- if (parsed.done) {
606
- const trailing = extractTrailingProse(raw);
607
- if (trailing) {
608
- finalMessage = [parsed.message.trim(), trailing]
609
- .filter(Boolean)
610
- .join("\n\n");
611
- usedTrailingProse = true;
612
- }
613
- }
614
-
615
1746
  // Record the assistant's turn in the canonical transcript. The raw XML is
616
1747
  // the web rendering; the transcript keeps the plain progress message.
617
1748
  if (finalMessage?.trim()) {
618
- await this.session.appendTranscriptItem(
1749
+ await this.#appendHandoffTranscript(
1750
+ activeHandoff,
1751
+ "assistant",
619
1752
  assistantMessage(finalMessage),
620
1753
  );
621
1754
  }
622
1755
 
623
1756
  if (parsed.done) {
624
- if (!finalMessage.trim()) {
625
- const error = new ProtocolError(
626
- "done=true requires a non-empty final message.",
627
- );
628
- await this.emit("protocol.invalid", { message: error.message });
629
- await this.sendMessage(serializeProtocolError(error));
630
- continue;
631
- }
632
- // Some models (DeepSeek especially) write their tool request inside
633
- // the <message> of a done=true envelope instead of a <tool_call>
634
- // element. Completing there would swallow the tool call and end the
635
- // run with raw XML as the "answer". Treat it as a format slip: the
636
- // protocol-error feedback tells the model where tool calls go, and
637
- // the normal retry limit still applies.
638
- if (
639
- !parsed.toolCall
640
- && /<tool_calls[\s>]|<tool_call[\s>]|<invoke[\s>]/i.test(finalMessage)
641
- ) {
642
- const error = new ProtocolError(
643
- "Tool calls must use the <tool_call> element, not the <message> text.",
644
- );
645
- protocolErrors += 1;
646
- await this.emit("protocol.invalid", {
647
- message: error.message,
648
- count: protocolErrors,
649
- });
650
- if (protocolErrors >= this.limits.maxProtocolErrors) {
651
- throw new ProtocolError(
652
- `Protocol failed ${protocolErrors} consecutive times: ${error.message}`,
653
- );
654
- }
655
- await this.sendMessage(serializeProtocolError(error));
656
- continue;
657
- }
658
1757
  // done=true completes the run. A request may be answered directly
659
1758
  // (no tool call) or after any number of tools; the runtime does not
660
1759
  // second-guess whether "enough" work happened — that is the model's
661
1760
  // and the user's call, not a keyword heuristic.
662
- await this.session.update({
663
- phase: "idle",
664
- lastMessage: finalMessage,
665
- pendingToolResult: null,
666
- });
1761
+ await this.#queueSessionOperation(() => (
1762
+ this.session.clearPendingAssistantTurn(
1763
+ activeHandoff.handoffId,
1764
+ {
1765
+ phase: "idle",
1766
+ lastMessage: finalMessage,
1767
+ pendingToolResult: null,
1768
+ },
1769
+ )
1770
+ ));
667
1771
  await this.emit("run.completed", {
668
1772
  message: finalMessage,
669
1773
  usedTrailingProse,
@@ -692,6 +1796,11 @@ export class AgentRuntime {
692
1796
  + "If you need to take action on the user's machine, request one local tool. "
693
1797
  + "If you need information from the user, ask one specific question in <message> "
694
1798
  + "and set <done>true</done> so control returns to the user.",
1799
+ {
1800
+ allowAssistantContinuation: true,
1801
+ outboundKind: "progress_correction",
1802
+ runtimeTurn: turnNumber + 1,
1803
+ },
695
1804
  );
696
1805
  continue;
697
1806
  }
@@ -734,16 +1843,24 @@ export class AgentRuntime {
734
1843
  await this.onEvent?.(completionEvent);
735
1844
  // Record the rejected call and its error output so the transcript stays
736
1845
  // a faithful, replay-free record of what the model attempted.
737
- await this.session.appendTranscriptItem(functionCall({
738
- name: normalizedCall.name,
739
- args: normalizedCall.args,
740
- callId: normalizedCall.id,
741
- }));
742
- await this.session.appendTranscriptItem(functionCallOutput({
743
- callId: result.callId,
744
- output: toolResultOutput(result),
745
- }));
746
- await this.sendToolResult(result);
1846
+ await this.#appendHandoffTranscript(
1847
+ activeHandoff,
1848
+ "function_call",
1849
+ functionCall({
1850
+ name: normalizedCall.name,
1851
+ args: normalizedCall.args,
1852
+ callId: normalizedCall.id,
1853
+ }),
1854
+ );
1855
+ await this.#appendHandoffTranscript(
1856
+ activeHandoff,
1857
+ "function_output",
1858
+ functionCallOutput({
1859
+ callId: result.callId,
1860
+ output: toolResultOutput(result),
1861
+ }),
1862
+ );
1863
+ await this.sendToolResult(result, { runtimeTurn: turnNumber + 1 });
747
1864
  awaitingPendingAcknowledgement = true;
748
1865
  await this.emit("tool.result_sent", {
749
1866
  id: result.callId,
@@ -757,16 +1874,21 @@ export class AgentRuntime {
757
1874
  name: preparedCall.name,
758
1875
  args: preparedCall.args,
759
1876
  });
760
- await this.session.appendTranscriptItem(functionCall({
761
- name: preparedCall.name,
762
- args: preparedCall.args,
763
- callId: preparedCall.id,
764
- }));
1877
+ await this.#appendHandoffTranscript(
1878
+ activeHandoff,
1879
+ "function_call",
1880
+ functionCall({
1881
+ name: preparedCall.name,
1882
+ args: preparedCall.args,
1883
+ callId: preparedCall.id,
1884
+ }),
1885
+ );
765
1886
 
766
1887
  const isReadTool = preparedCall.definition.risk === "read";
767
1888
  const sideEffect = isReadTool ? null : identity;
768
1889
  const fingerprint = identity.fingerprint;
769
1890
  let result;
1891
+ let restoredForTool = false;
770
1892
 
771
1893
  if (
772
1894
  sideEffect
@@ -885,32 +2007,46 @@ export class AgentRuntime {
885
2007
  }
886
2008
 
887
2009
  if (!result) {
2010
+ if (preparedCall.definition.requiresVisibleBrowser) {
2011
+ await this.adapter.restoreWindow?.();
2012
+ restoredForTool = true;
2013
+ }
888
2014
  await this.emit("tool.started", {
889
2015
  id: preparedCall.id,
890
2016
  name: preparedCall.name,
891
2017
  });
892
- result = await this.registry.execute(preparedCall, {
893
- projectRoot,
894
- allowOutside: grants?.allowOutside ?? false,
895
- toolTimeoutMs: this.limits.toolTimeoutMs,
896
- onToolOutput: async (output) => {
897
- await this.session.appendToolOutput({
898
- id: preparedCall.id,
899
- name: preparedCall.name,
900
- ...output,
901
- });
902
- await this.onEvent?.({
903
- type: "tool.output",
904
- sessionId: this.session.sessionId,
905
- timestamp: new Date().toISOString(),
906
- payload: {
2018
+ try {
2019
+ result = await this.registry.execute(preparedCall, {
2020
+ projectRoot,
2021
+ allowOutside: grants?.allowOutside ?? false,
2022
+ toolTimeoutMs: this.limits.toolTimeoutMs,
2023
+ sessionId: this.session.sessionId,
2024
+ operationKey: identity.operationKey,
2025
+ requestSignature: identity.requestSignature,
2026
+ runAuxiliaryTurn: (options) => this.runAuxiliaryTurn(options),
2027
+ onToolOutput: async (output) => {
2028
+ await this.session.appendToolOutput({
907
2029
  id: preparedCall.id,
908
2030
  name: preparedCall.name,
909
2031
  ...output,
910
- },
911
- });
912
- },
913
- });
2032
+ });
2033
+ await this.onEvent?.({
2034
+ type: "tool.output",
2035
+ sessionId: this.session.sessionId,
2036
+ timestamp: new Date().toISOString(),
2037
+ payload: {
2038
+ id: preparedCall.id,
2039
+ name: preparedCall.name,
2040
+ ...output,
2041
+ },
2042
+ });
2043
+ },
2044
+ });
2045
+ } finally {
2046
+ if (restoredForTool) {
2047
+ await this.adapter.page?.bringToFront?.().catch(() => null);
2048
+ }
2049
+ }
914
2050
  }
915
2051
 
916
2052
  if (sideEffect) {
@@ -933,17 +2069,26 @@ export class AgentRuntime {
933
2069
  }
934
2070
  }
935
2071
 
936
- await this.session.appendTranscriptItem(functionCallOutput({
937
- callId: result.callId,
938
- output: toolResultOutput(result),
939
- }));
940
- await this.sendToolResult(result);
2072
+ this.#assertProviderReadyForMessage();
2073
+
2074
+ await this.#appendHandoffTranscript(
2075
+ activeHandoff,
2076
+ "function_output",
2077
+ functionCallOutput({
2078
+ callId: result.callId,
2079
+ output: toolResultOutput(result),
2080
+ }),
2081
+ );
2082
+ await this.sendToolResult(result, { runtimeTurn: turnNumber + 1 });
941
2083
  awaitingPendingAcknowledgement = true;
942
2084
  await this.emit("tool.result_sent", {
943
2085
  id: result.callId,
944
2086
  name: result.name,
945
2087
  ok: result.ok,
946
2088
  });
2089
+ if (restoredForTool) {
2090
+ await this.adapter.minimizeWindow?.();
2091
+ }
947
2092
  }
948
2093
 
949
2094
  }