wtagent 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -9
- package/package.json +9 -8
- package/src/artifacts/artifact-store.js +154 -0
- package/src/audio/music-artifact.js +66 -0
- package/src/audio/native-music-receiver.js +91 -0
- package/src/browser/base-web-adapter.js +2418 -201
- package/src/browser/cdp-browser.js +223 -25
- package/src/browser/cdp-state.js +8 -5
- package/src/browser/chatgpt-dom.js +179 -0
- package/src/browser/chatgpt-web-adapter.js +831 -69
- package/src/browser/claude-web-adapter.js +13 -0
- package/src/browser/fake-web-model-adapter.js +152 -3
- package/src/browser/gemini-web-adapter.js +71 -20
- package/src/browser/glm-web-adapter.js +2 -2
- package/src/browser/grok-web-adapter.js +35 -3
- package/src/browser/rendered-text.js +11 -0
- package/src/cli/i18n.js +50 -20
- package/src/cli/main.js +124 -40
- package/src/cli/prompt-input.js +68 -0
- package/src/image/browser-image-driver.js +238 -0
- package/src/image/browser-image-session-pool.js +96 -0
- package/src/image/image-generation-service.js +101 -0
- package/src/image/image-provider-driver.js +22 -0
- package/src/image/native-image-receiver.js +66 -0
- package/src/image/provider-registry.js +23 -0
- package/src/image/providers/chatgpt-image-driver.js +211 -0
- package/src/image/providers/gemini-image-driver.js +100 -0
- package/src/image/providers/grok-image-driver.js +193 -0
- package/src/platform/command-launcher.js +1 -1
- package/src/platform/paths.js +19 -0
- package/src/platform/windows-diagnostics.js +9 -16
- package/src/policy/path-guard.js +5 -1
- package/src/policy/policy-engine.js +10 -5
- package/src/protocol/envelope.js +74 -0
- package/src/protocol/markers.js +16 -0
- package/src/protocol/prompt-builder.js +12 -9
- package/src/protocol/xml-protocol.js +36 -18
- package/src/runtime/agent-runtime.js +1383 -238
- package/src/session/agent-session.js +1047 -121
- package/src/tools/default-tools.js +1 -1
- package/src/tools/image-tools.js +92 -0
- package/src/tools/registry.js +13 -1
- package/docs/technical-design.md +0 -866
|
@@ -1,13 +1,16 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import {
|
|
3
3
|
cdata,
|
|
4
4
|
extractTrailingProse,
|
|
5
5
|
parseAgentResponse,
|
|
6
6
|
serializeProtocolError,
|
|
7
7
|
serializeToolResult,
|
|
8
|
-
stripUiNoiseLines,
|
|
9
8
|
} from "../protocol/xml-protocol.js";
|
|
10
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
appendSystemReminder,
|
|
11
|
+
DEFAULT_SYSTEM_REMINDER,
|
|
12
|
+
NATIVE_IMAGE_SYSTEM_REMINDER,
|
|
13
|
+
} from "../protocol/markers.js";
|
|
11
14
|
import {
|
|
12
15
|
buildBootstrapPrompt,
|
|
13
16
|
buildResumePrompt,
|
|
@@ -16,17 +19,22 @@ import {
|
|
|
16
19
|
assistantMessage,
|
|
17
20
|
functionCall,
|
|
18
21
|
functionCallOutput,
|
|
22
|
+
messageText,
|
|
19
23
|
toolResultOutput,
|
|
20
24
|
userMessage,
|
|
21
25
|
} from "../session/canonical-transcript.js";
|
|
22
26
|
import { DEFAULT_LIMITS } from "../shared/limits.js";
|
|
27
|
+
import { MUSIC_SYSTEM_REMINDER } from "../audio/native-music-receiver.js";
|
|
23
28
|
import { utf8ByteLength } from "../shared/text-budget.js";
|
|
24
29
|
import {
|
|
25
30
|
BrowserAdapterError,
|
|
26
31
|
ProtocolError,
|
|
27
32
|
ToolValidationError,
|
|
28
33
|
} from "../shared/errors.js";
|
|
29
|
-
import {
|
|
34
|
+
import {
|
|
35
|
+
isConnectionLostError,
|
|
36
|
+
isValidOutboundCorrelationId,
|
|
37
|
+
} from "../browser/base-web-adapter.js";
|
|
30
38
|
import { isUsageLimitNotice } from "../shared/usage-limit.js";
|
|
31
39
|
|
|
32
40
|
const EMPTY_ASSISTANT_CONTINUE_MESSAGE =
|
|
@@ -44,6 +52,11 @@ const GENERATION_FAILED_CONTINUE_MESSAGE =
|
|
|
44
52
|
+ "Retry the immediately preceding task from the existing conversation context. Do not "
|
|
45
53
|
+ "repeat any local tool operation whose result is already present. Reply using the "
|
|
46
54
|
+ "required <agent_response> XML protocol.";
|
|
55
|
+
const OUTBOUND_CORRELATION_PREFIX =
|
|
56
|
+
" Opaque WTAgent transport correlation ID (do not repeat): ";
|
|
57
|
+
const OUTBOUND_CORRELATION_RESERVE_BYTES = utf8ByteLength(
|
|
58
|
+
`${OUTBOUND_CORRELATION_PREFIX}${"0".repeat(36)}.`,
|
|
59
|
+
);
|
|
47
60
|
|
|
48
61
|
function canonicalize(value) {
|
|
49
62
|
if (Array.isArray(value)) {
|
|
@@ -145,6 +158,8 @@ export class AgentRuntime {
|
|
|
145
158
|
approval,
|
|
146
159
|
onEvent,
|
|
147
160
|
postAuthSetup = null,
|
|
161
|
+
nativeImageReceiver = null,
|
|
162
|
+
nativeMusicReceiver = null,
|
|
148
163
|
limits = DEFAULT_LIMITS,
|
|
149
164
|
}) {
|
|
150
165
|
this.adapter = adapter;
|
|
@@ -154,7 +169,14 @@ export class AgentRuntime {
|
|
|
154
169
|
this.approval = approval;
|
|
155
170
|
this.onEvent = onEvent;
|
|
156
171
|
this.postAuthSetup = postAuthSetup;
|
|
172
|
+
this.nativeImageReceiver = nativeImageReceiver;
|
|
173
|
+
this.nativeMusicReceiver = nativeMusicReceiver;
|
|
157
174
|
this.limits = limits;
|
|
175
|
+
this.conversationIdentityQueue = Promise.resolve();
|
|
176
|
+
this.deferConversationIdentityPersistence = false;
|
|
177
|
+
this.adapter.setConversationIdentityListener?.((identity) =>
|
|
178
|
+
this.#queueConversationIdentity(identity)
|
|
179
|
+
);
|
|
158
180
|
}
|
|
159
181
|
|
|
160
182
|
async emit(type, payload = {}) {
|
|
@@ -163,60 +185,438 @@ export class AgentRuntime {
|
|
|
163
185
|
return event;
|
|
164
186
|
}
|
|
165
187
|
|
|
166
|
-
|
|
167
|
-
|
|
188
|
+
#assertProviderReadyForMessage() {
|
|
189
|
+
if (this.session.state.pendingAuxiliaryTurn != null) {
|
|
190
|
+
throw new BrowserAdapterError(
|
|
191
|
+
"The provider operation has no verified assistant boundary. No further message was sent.",
|
|
192
|
+
{ code: "AUXILIARY_TURN_RECOVERY_REQUIRED", recoverable: false },
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
if (this.session.state.pendingOutbound != null) {
|
|
196
|
+
throw new BrowserAdapterError(
|
|
197
|
+
"A previous provider message has not been reconciled. No further message was sent.",
|
|
198
|
+
{ code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
async sendMessage(text, {
|
|
204
|
+
files = [],
|
|
205
|
+
maxBytes = null,
|
|
206
|
+
allowFreshReconnect = false,
|
|
207
|
+
allowAssistantContinuation = false,
|
|
208
|
+
requireAttachments = false,
|
|
209
|
+
outboundKind = "runtime_message",
|
|
210
|
+
transcriptItems = [],
|
|
211
|
+
runtimeTurn = null,
|
|
212
|
+
pendingToolAcknowledgement = null,
|
|
213
|
+
} = {}) {
|
|
214
|
+
this.#assertProviderReadyForMessage();
|
|
215
|
+
await this.nativeMusicReceiver?.prepare();
|
|
216
|
+
const outboundId = randomUUID();
|
|
217
|
+
const reminder = this.nativeMusicReceiver ? MUSIC_SYSTEM_REMINDER
|
|
218
|
+
: this.nativeImageReceiver ? NATIVE_IMAGE_SYSTEM_REMINDER : DEFAULT_SYSTEM_REMINDER;
|
|
219
|
+
const message = appendSystemReminder(
|
|
220
|
+
text,
|
|
221
|
+
`${reminder}${OUTBOUND_CORRELATION_PREFIX}${outboundId}.`,
|
|
222
|
+
);
|
|
223
|
+
const pendingOutbound = {
|
|
224
|
+
kind: outboundKind,
|
|
225
|
+
outboundId,
|
|
226
|
+
messageHash: createHash("sha256").update(message).digest("hex"),
|
|
227
|
+
preparedAt: new Date().toISOString(),
|
|
228
|
+
transcriptItems,
|
|
229
|
+
runtimeTurn: Number.isSafeInteger(runtimeTurn) ? runtimeTurn : null,
|
|
230
|
+
pendingToolAcknowledgement,
|
|
231
|
+
conversationUrl: this.session.state.conversationUrl ?? null,
|
|
232
|
+
conversationTargetId: this.session.state.conversationTargetId ?? null,
|
|
233
|
+
};
|
|
234
|
+
await this.#queueSessionUpdate({ pendingOutbound });
|
|
168
235
|
try {
|
|
169
|
-
await this.#sendMessageWithReconnect(message, {
|
|
236
|
+
const sendResult = await this.#sendMessageWithReconnect(message, {
|
|
237
|
+
files,
|
|
238
|
+
maxBytes,
|
|
239
|
+
allowFreshReconnect,
|
|
240
|
+
allowAssistantContinuation,
|
|
241
|
+
requireAttachments,
|
|
242
|
+
outboundId,
|
|
243
|
+
});
|
|
244
|
+
const handoff = await this.#queueSessionOperation(() => (
|
|
245
|
+
this.session.commitPendingOutboundHandoff({
|
|
246
|
+
outboundId,
|
|
247
|
+
handoff: {
|
|
248
|
+
runtimeTurn: pendingOutbound.runtimeTurn
|
|
249
|
+
?? this.session.state.turn,
|
|
250
|
+
conversationUrl: sendResult?.conversationUrl
|
|
251
|
+
?? this.session.state.conversationUrl,
|
|
252
|
+
conversationTargetId: sendResult?.conversationTargetId
|
|
253
|
+
?? this.session.state.conversationTargetId,
|
|
254
|
+
userMessageId: sendResult?.userMessageId ?? null,
|
|
255
|
+
userTurn: sendResult?.userTurn ?? null,
|
|
256
|
+
assistantBaseline: sendResult?.assistantBaseline ?? null,
|
|
257
|
+
preOutboundMarkerIds: sendResult?.preOutboundMarkerIds ?? [],
|
|
258
|
+
pendingToolAcknowledgement,
|
|
259
|
+
},
|
|
260
|
+
})
|
|
261
|
+
));
|
|
262
|
+
return { ...sendResult, pendingAssistantTurn: handoff };
|
|
263
|
+
} catch (error) {
|
|
264
|
+
const sendStatus = this.adapter.getLastSendStatus?.() ?? "commit-unknown";
|
|
265
|
+
const nextPending = sendStatus === "not-submitted"
|
|
266
|
+
? null
|
|
267
|
+
: {
|
|
268
|
+
...pendingOutbound,
|
|
269
|
+
status: "commit-unknown",
|
|
270
|
+
failedAt: new Date().toISOString(),
|
|
271
|
+
};
|
|
272
|
+
await this.#queueSessionOperation(async () => {
|
|
273
|
+
if (this.session.state.pendingOutbound?.outboundId === outboundId) {
|
|
274
|
+
await this.session.update({ pendingOutbound: nextPending });
|
|
275
|
+
}
|
|
276
|
+
}).catch(() => {});
|
|
277
|
+
throw error;
|
|
170
278
|
} finally {
|
|
171
|
-
|
|
279
|
+
// URL observation must never replace the send result. Top-frame navigation
|
|
280
|
+
// is persisted eagerly; this is only a best-effort final checkpoint.
|
|
281
|
+
const identity = await this.#syncConversationIdentity({
|
|
282
|
+
suppressErrors: true,
|
|
283
|
+
});
|
|
284
|
+
await this.#refreshPendingOutboundIdentity(outboundId, identity)
|
|
172
285
|
.catch(() => null);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
async #persistConversationIdentity(identity, { allowFresh = false } = {}) {
|
|
290
|
+
const conversationUrl = identity?.conversationUrl ?? null;
|
|
291
|
+
const kind = identity?.kind
|
|
292
|
+
?? (conversationUrl && this.adapter.classifyConversationUrl?.(conversationUrl));
|
|
293
|
+
if (
|
|
294
|
+
!conversationUrl
|
|
295
|
+
|| (
|
|
296
|
+
!["restorable", "provisional"].includes(kind)
|
|
297
|
+
&& !(allowFresh && kind === "fresh")
|
|
298
|
+
)
|
|
299
|
+
) {
|
|
300
|
+
return null;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const targetId = typeof identity?.targetId === "string"
|
|
304
|
+
&& identity.targetId
|
|
305
|
+
? identity.targetId
|
|
306
|
+
: null;
|
|
307
|
+
const patch = {};
|
|
308
|
+
if (conversationUrl !== this.session.state.conversationUrl) {
|
|
309
|
+
patch.conversationUrl = conversationUrl;
|
|
310
|
+
}
|
|
311
|
+
if (
|
|
312
|
+
targetId
|
|
313
|
+
&& targetId !== this.session.state.conversationTargetId
|
|
314
|
+
) {
|
|
315
|
+
patch.conversationTargetId = targetId;
|
|
316
|
+
}
|
|
317
|
+
if (Object.keys(patch).length > 0) {
|
|
318
|
+
await this.session.update(patch);
|
|
319
|
+
}
|
|
320
|
+
return {
|
|
321
|
+
conversationUrl,
|
|
322
|
+
targetId: targetId ?? this.session.state.conversationTargetId ?? null,
|
|
323
|
+
kind,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
#queueConversationIdentity(identity, options = {}) {
|
|
328
|
+
if (this.deferConversationIdentityPersistence) {
|
|
329
|
+
return Promise.resolve(null);
|
|
330
|
+
}
|
|
331
|
+
const operation = this.conversationIdentityQueue
|
|
332
|
+
.catch(() => null)
|
|
333
|
+
.then(() => this.#persistConversationIdentity(identity, options));
|
|
334
|
+
this.conversationIdentityQueue = operation;
|
|
335
|
+
return operation;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
#queueSessionOperation(callback) {
|
|
339
|
+
const operation = this.conversationIdentityQueue
|
|
340
|
+
.catch(() => null)
|
|
341
|
+
.then(callback);
|
|
342
|
+
this.conversationIdentityQueue = operation;
|
|
343
|
+
return operation;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
#queueSessionUpdate(patch) {
|
|
347
|
+
return this.#queueSessionOperation(() => this.session.update(patch));
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
async #refreshPendingOutboundIdentity(
|
|
351
|
+
outboundId,
|
|
352
|
+
identity = null,
|
|
353
|
+
{ required = false } = {},
|
|
354
|
+
) {
|
|
355
|
+
return await this.#queueSessionOperation(async () => {
|
|
356
|
+
const current = this.session.state.pendingOutbound;
|
|
357
|
+
if (!current || current.outboundId !== outboundId) {
|
|
358
|
+
if (required) {
|
|
359
|
+
throw new BrowserAdapterError(
|
|
360
|
+
"The pending outbound changed before its reconnect target could be checkpointed.",
|
|
361
|
+
{ code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
return null;
|
|
365
|
+
}
|
|
366
|
+
const conversationUrl = identity?.conversationUrl
|
|
367
|
+
?? this.session.state.conversationUrl
|
|
368
|
+
?? current.conversationUrl
|
|
369
|
+
?? null;
|
|
370
|
+
const conversationTargetId = identity?.targetId
|
|
371
|
+
?? identity?.conversationTargetId
|
|
372
|
+
?? this.session.state.conversationTargetId
|
|
373
|
+
?? current.conversationTargetId
|
|
374
|
+
?? null;
|
|
173
375
|
if (
|
|
174
|
-
conversationUrl
|
|
175
|
-
&&
|
|
376
|
+
conversationUrl === current.conversationUrl
|
|
377
|
+
&& conversationTargetId === current.conversationTargetId
|
|
176
378
|
) {
|
|
177
|
-
|
|
379
|
+
return structuredClone(current);
|
|
380
|
+
}
|
|
381
|
+
const refreshed = {
|
|
382
|
+
...current,
|
|
383
|
+
conversationUrl,
|
|
384
|
+
conversationTargetId,
|
|
385
|
+
};
|
|
386
|
+
await this.session.update({
|
|
387
|
+
pendingOutbound: refreshed,
|
|
388
|
+
conversationUrl,
|
|
389
|
+
conversationTargetId,
|
|
390
|
+
});
|
|
391
|
+
return structuredClone(refreshed);
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
async #syncConversationIdentity({
|
|
396
|
+
suppressErrors = false,
|
|
397
|
+
allowFresh = false,
|
|
398
|
+
identity = null,
|
|
399
|
+
} = {}) {
|
|
400
|
+
// Observe and persist inside one queue entry. If a navigation callback fires
|
|
401
|
+
// while observation awaits the adapter, it queues after this operation and
|
|
402
|
+
// remains authoritative instead of being overwritten by a stale snapshot.
|
|
403
|
+
const operation = this.conversationIdentityQueue
|
|
404
|
+
.catch(() => null)
|
|
405
|
+
.then(async () => {
|
|
406
|
+
let observed = identity;
|
|
407
|
+
if (!observed) {
|
|
408
|
+
if (typeof this.adapter.getConversationIdentity === "function") {
|
|
409
|
+
observed = await this.adapter.getConversationIdentity();
|
|
410
|
+
} else {
|
|
411
|
+
const conversationUrl = await this.adapter.getConversationUrl();
|
|
412
|
+
observed = {
|
|
413
|
+
conversationUrl,
|
|
414
|
+
targetId: null,
|
|
415
|
+
kind: this.adapter.classifyConversationUrl?.(conversationUrl)
|
|
416
|
+
?? (/\/c\//.test(new URL(conversationUrl).pathname)
|
|
417
|
+
? "restorable"
|
|
418
|
+
: "fresh"),
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
return await this.#persistConversationIdentity(observed, { allowFresh });
|
|
423
|
+
});
|
|
424
|
+
this.conversationIdentityQueue = operation;
|
|
425
|
+
try {
|
|
426
|
+
return await operation;
|
|
427
|
+
} catch (error) {
|
|
428
|
+
if (!suppressErrors) {
|
|
429
|
+
throw error;
|
|
178
430
|
}
|
|
431
|
+
return null;
|
|
179
432
|
}
|
|
180
433
|
}
|
|
181
434
|
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
// the
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
435
|
+
// Retry only when the adapter proves submission never started. Once the Send
|
|
436
|
+
// action has occurred, a dead transport or missing bubble is commit-unknown;
|
|
437
|
+
// replaying the payload could duplicate provider work and attachments.
|
|
438
|
+
async #sendMessageWithReconnect(message, {
|
|
439
|
+
files,
|
|
440
|
+
maxBytes,
|
|
441
|
+
allowFreshReconnect,
|
|
442
|
+
allowAssistantContinuation,
|
|
443
|
+
requireAttachments,
|
|
444
|
+
outboundId,
|
|
445
|
+
}) {
|
|
188
446
|
try {
|
|
189
|
-
await this.adapter.sendMessage(message, {
|
|
447
|
+
return await this.adapter.sendMessage(message, {
|
|
448
|
+
files,
|
|
449
|
+
maxBytes,
|
|
450
|
+
outboundId,
|
|
451
|
+
allowAssistantContinuation,
|
|
452
|
+
requireAttachments,
|
|
453
|
+
});
|
|
190
454
|
} catch (error) {
|
|
191
455
|
if (isConnectionLostError(error)) {
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
456
|
+
const sendStatus = this.adapter.getLastSendStatus?.() ?? "commit-unknown";
|
|
457
|
+
if (sendStatus !== "not-submitted") {
|
|
458
|
+
throw new BrowserAdapterError(
|
|
459
|
+
`${this.adapter.providerName ?? "Web provider"} connection was lost after submission may have started. `
|
|
460
|
+
+ "The message was not sent again automatically.",
|
|
461
|
+
{
|
|
462
|
+
code: "SEND_COMMIT_UNKNOWN",
|
|
463
|
+
recoverable: false,
|
|
464
|
+
cause: error,
|
|
465
|
+
},
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
await this.#reconnectAndRestore({
|
|
469
|
+
allowFresh: allowFreshReconnect,
|
|
470
|
+
checkpointIdentity: (restoration) => (
|
|
471
|
+
this.#refreshPendingOutboundIdentity(
|
|
472
|
+
outboundId,
|
|
473
|
+
restoration,
|
|
474
|
+
{ required: true },
|
|
475
|
+
)
|
|
476
|
+
),
|
|
477
|
+
});
|
|
478
|
+
await this.nativeMusicReceiver?.prepare();
|
|
479
|
+
return await this.adapter.sendMessage(message, {
|
|
480
|
+
files,
|
|
481
|
+
maxBytes,
|
|
482
|
+
outboundId,
|
|
483
|
+
allowAssistantContinuation,
|
|
484
|
+
requireAttachments,
|
|
485
|
+
});
|
|
199
486
|
}
|
|
200
487
|
throw error;
|
|
201
488
|
}
|
|
202
489
|
}
|
|
203
490
|
|
|
204
|
-
// Reconnects to the still-alive Chrome and
|
|
205
|
-
//
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
491
|
+
// Reconnects to the still-alive Chrome and restores the exact page target when
|
|
492
|
+
// possible. Once a context-dependent message has been sent, landing on a fresh
|
|
493
|
+
// page is never a safe restoration.
|
|
494
|
+
async #reconnectAndRestore({
|
|
495
|
+
allowFresh = false,
|
|
496
|
+
checkpointIdentity = null,
|
|
497
|
+
} = {}) {
|
|
498
|
+
// Navigation callbacks must not persist a replacement root target without
|
|
499
|
+
// the pending record that causally owns it. Suppress them until the caller's
|
|
500
|
+
// checkpoint writes both identities in one Session transition.
|
|
501
|
+
this.deferConversationIdentityPersistence = true;
|
|
502
|
+
try {
|
|
503
|
+
await this.conversationIdentityQueue.catch(() => null);
|
|
504
|
+
await this.adapter.reconnect?.(
|
|
505
|
+
this.session.state.conversationUrl,
|
|
506
|
+
{ preferredTargetId: this.session.state.conversationTargetId ?? null },
|
|
507
|
+
);
|
|
508
|
+
const restoration = await this.adapter.startConversation(
|
|
509
|
+
this.session.state.conversationUrl,
|
|
510
|
+
{
|
|
511
|
+
expectedAssistantMessageId: this.session.state.lastAssistantMessageId
|
|
512
|
+
?? null,
|
|
513
|
+
expectedUserMessageId: this.session.state.lastUserMessageId ?? null,
|
|
514
|
+
},
|
|
515
|
+
);
|
|
516
|
+
if (
|
|
517
|
+
restoration?.status !== "restored-existing"
|
|
518
|
+
&& !(allowFresh && restoration?.status === "verified-fresh")
|
|
519
|
+
) {
|
|
520
|
+
throw new BrowserAdapterError(
|
|
521
|
+
`${this.adapter.providerName} conversation disappeared while a turn was in progress. `
|
|
522
|
+
+ "Refusing to continue on a blank page; resume the session to recover safely.",
|
|
523
|
+
{
|
|
524
|
+
code: "CONVERSATION_RESTORE_REQUIRED",
|
|
525
|
+
recoverable: false,
|
|
526
|
+
details: { restorationStatus: restoration?.status ?? null },
|
|
527
|
+
},
|
|
528
|
+
);
|
|
529
|
+
}
|
|
530
|
+
const checkpoint = checkpointIdentity
|
|
531
|
+
? await checkpointIdentity(restoration)
|
|
532
|
+
: null;
|
|
533
|
+
return { restoration, checkpoint };
|
|
534
|
+
} finally {
|
|
535
|
+
this.deferConversationIdentityPersistence = false;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
async #freshRebuildBlockers({ instruction = null } = {}) {
|
|
540
|
+
const state = this.session.state;
|
|
541
|
+
const blockers = [];
|
|
542
|
+
if (state.pendingOutbound != null) {
|
|
543
|
+
blockers.push("unconfirmed outbound message");
|
|
544
|
+
}
|
|
545
|
+
if (state.pendingAssistantTurn != null) {
|
|
546
|
+
blockers.push("pending assistant handoff");
|
|
547
|
+
}
|
|
548
|
+
if (state.pendingAuxiliaryTurn != null) {
|
|
549
|
+
blockers.push("pending auxiliary provider turn");
|
|
550
|
+
}
|
|
551
|
+
if (state.pendingToolResult != null) {
|
|
552
|
+
blockers.push("pending tool result");
|
|
553
|
+
}
|
|
554
|
+
if (Object.keys(state.completedTools ?? {}).length > 0) {
|
|
555
|
+
blockers.push("completed tool history");
|
|
556
|
+
}
|
|
557
|
+
if (Object.keys(state.sideEffectTools ?? {}).length > 0) {
|
|
558
|
+
blockers.push("side-effect history");
|
|
559
|
+
}
|
|
560
|
+
if (state.lastMessage != null) {
|
|
561
|
+
blockers.push("completed assistant result");
|
|
562
|
+
}
|
|
563
|
+
if (
|
|
564
|
+
state.lastAssistantMessageId != null
|
|
565
|
+
|| await this.session.hasEvent("model.message_complete")
|
|
566
|
+
) {
|
|
567
|
+
blockers.push("completed or ambiguous assistant response");
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
const suppliedInstruction = instruction?.trim() ?? "";
|
|
571
|
+
const followUps = state.followUps ?? [];
|
|
572
|
+
const persistedInstructions = [
|
|
573
|
+
...new Set(followUps.map((item) => item?.instruction?.trim()).filter(Boolean)),
|
|
574
|
+
];
|
|
575
|
+
const followUpsHaveAttachments = followUps.some((item) =>
|
|
576
|
+
item?.attachments != null
|
|
577
|
+
&& (!Array.isArray(item.attachments) || item.attachments.length > 0)
|
|
214
578
|
);
|
|
579
|
+
if (
|
|
580
|
+
followUpsHaveAttachments
|
|
581
|
+
|| persistedInstructions.length > 1
|
|
582
|
+
|| (
|
|
583
|
+
suppliedInstruction
|
|
584
|
+
&& persistedInstructions.length === 1
|
|
585
|
+
&& persistedInstructions[0] !== suppliedInstruction
|
|
586
|
+
)
|
|
587
|
+
) {
|
|
588
|
+
blockers.push("earlier follow-up history");
|
|
589
|
+
}
|
|
590
|
+
const rebuildInstruction = suppliedInstruction
|
|
591
|
+
|| (persistedInstructions.length === 1 ? persistedInstructions[0] : null);
|
|
592
|
+
|
|
593
|
+
const transcript = await this.session.readTranscript();
|
|
594
|
+
if (transcript.items.length !== 1) {
|
|
595
|
+
blockers.push(
|
|
596
|
+
transcript.items.length === 0
|
|
597
|
+
? "missing opening transcript"
|
|
598
|
+
: "canonical transcript history",
|
|
599
|
+
);
|
|
600
|
+
} else {
|
|
601
|
+
const item = transcript.items[0].item;
|
|
602
|
+
const hasNoAttachments = item?.attachments == null
|
|
603
|
+
|| (Array.isArray(item.attachments) && item.attachments.length === 0);
|
|
604
|
+
const isOriginalUser = item?.type === "message"
|
|
605
|
+
&& item.role === "user"
|
|
606
|
+
&& messageText(item) === String(state.task ?? "")
|
|
607
|
+
&& hasNoAttachments;
|
|
608
|
+
if (!isOriginalUser) {
|
|
609
|
+
blockers.push("canonical transcript history");
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
return { blockers, transcript, rebuildInstruction };
|
|
215
613
|
}
|
|
216
614
|
|
|
217
615
|
buildToolResultMessage(result, { suffix = "" } = {}) {
|
|
218
616
|
const limit = this.limits.maxBrowserToolResultBytes;
|
|
219
|
-
const nonResultBytes = utf8ByteLength(appendSystemReminder(suffix
|
|
617
|
+
const nonResultBytes = utf8ByteLength(appendSystemReminder(suffix,
|
|
618
|
+
this.nativeImageReceiver ? NATIVE_IMAGE_SYSTEM_REMINDER : DEFAULT_SYSTEM_REMINDER))
|
|
619
|
+
+ OUTBOUND_CORRELATION_RESERVE_BYTES;
|
|
220
620
|
const resultBudget = limit - nonResultBytes;
|
|
221
621
|
if (resultBudget < 512) {
|
|
222
622
|
throw new RangeError(
|
|
@@ -226,10 +626,365 @@ export class AgentRuntime {
|
|
|
226
626
|
return `${serializeToolResult(result, { maxBytes: resultBudget })}${suffix}`;
|
|
227
627
|
}
|
|
228
628
|
|
|
229
|
-
async sendToolResult(result, {
|
|
230
|
-
|
|
231
|
-
|
|
629
|
+
async sendToolResult(result, {
|
|
630
|
+
suffix = "",
|
|
631
|
+
runtimeTurn = null,
|
|
632
|
+
} = {}) {
|
|
633
|
+
return await this.sendMessage(
|
|
634
|
+
this.buildToolResultMessage(result, { suffix }),
|
|
635
|
+
{
|
|
636
|
+
maxBytes: this.limits.maxBrowserToolResultBytes,
|
|
637
|
+
outboundKind: "tool_result",
|
|
638
|
+
runtimeTurn,
|
|
639
|
+
pendingToolAcknowledgement: {
|
|
640
|
+
callId: result.callId ?? null,
|
|
641
|
+
name: result.name ?? null,
|
|
642
|
+
operationSignature: result.operationSignature ?? null,
|
|
643
|
+
},
|
|
644
|
+
},
|
|
645
|
+
);
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// Runs a provider-native helper turn (currently used by image.generate) in
|
|
649
|
+
// the same web conversation without exposing that turn as another WTAgent
|
|
650
|
+
// protocol exchange. The send still goes through the adapter's full causal
|
|
651
|
+
// checks, and both provider message identities are checkpointed before the
|
|
652
|
+
// caller is allowed to send the tool result.
|
|
653
|
+
async runAuxiliaryTurn({
|
|
654
|
+
text,
|
|
655
|
+
files = [],
|
|
656
|
+
timeoutMs = this.limits.modelTurnTimeoutMs,
|
|
657
|
+
waitForCompletion,
|
|
658
|
+
}) {
|
|
659
|
+
if (typeof waitForCompletion !== "function") {
|
|
660
|
+
throw new TypeError("An auxiliary turn requires a completion callback.");
|
|
661
|
+
}
|
|
662
|
+
this.#assertProviderReadyForMessage();
|
|
663
|
+
const outboundId = randomUUID();
|
|
664
|
+
const message = appendSystemReminder(
|
|
665
|
+
text,
|
|
666
|
+
`This is an internal WTAgent provider operation. Perform only the requested operation; do not emit the WTAgent XML protocol.${OUTBOUND_CORRELATION_PREFIX}${outboundId}.`,
|
|
667
|
+
);
|
|
668
|
+
const pendingOutbound = {
|
|
669
|
+
kind: "auxiliary_tool_turn",
|
|
670
|
+
outboundId,
|
|
671
|
+
messageHash: createHash("sha256").update(message).digest("hex"),
|
|
672
|
+
preparedAt: new Date().toISOString(),
|
|
673
|
+
transcriptItems: [],
|
|
674
|
+
runtimeTurn: null,
|
|
675
|
+
pendingToolAcknowledgement: null,
|
|
676
|
+
conversationUrl: this.session.state.conversationUrl ?? null,
|
|
677
|
+
conversationTargetId: this.session.state.conversationTargetId ?? null,
|
|
678
|
+
};
|
|
679
|
+
await this.#queueSessionUpdate({ pendingOutbound });
|
|
680
|
+
|
|
681
|
+
let sendResult;
|
|
682
|
+
try {
|
|
683
|
+
sendResult = await this.#sendMessageWithReconnect(message, {
|
|
684
|
+
files,
|
|
685
|
+
maxBytes: null,
|
|
686
|
+
allowFreshReconnect: false,
|
|
687
|
+
allowAssistantContinuation: false,
|
|
688
|
+
requireAttachments: files.length > 0,
|
|
689
|
+
outboundId,
|
|
690
|
+
});
|
|
691
|
+
await this.#queueSessionOperation(() => (
|
|
692
|
+
this.session.commitPendingAuxiliaryTurn({
|
|
693
|
+
outboundId,
|
|
694
|
+
turn: {
|
|
695
|
+
conversationUrl: sendResult?.conversationUrl
|
|
696
|
+
?? this.session.state.conversationUrl,
|
|
697
|
+
conversationTargetId: sendResult?.conversationTargetId
|
|
698
|
+
?? this.session.state.conversationTargetId,
|
|
699
|
+
userMessageId: sendResult?.userMessageId ?? null,
|
|
700
|
+
userTurn: sendResult?.userTurn ?? null,
|
|
701
|
+
assistantBaseline: sendResult?.assistantBaseline ?? null,
|
|
702
|
+
},
|
|
703
|
+
})
|
|
704
|
+
));
|
|
705
|
+
} catch (error) {
|
|
706
|
+
const sendStatus = this.adapter.getLastSendStatus?.() ?? "commit-unknown";
|
|
707
|
+
await this.#queueSessionOperation(async () => {
|
|
708
|
+
if (this.session.state.pendingOutbound?.outboundId === outboundId) {
|
|
709
|
+
await this.session.update({
|
|
710
|
+
pendingOutbound: sendStatus === "not-submitted"
|
|
711
|
+
? null
|
|
712
|
+
: {
|
|
713
|
+
...pendingOutbound,
|
|
714
|
+
status: "commit-unknown",
|
|
715
|
+
failedAt: new Date().toISOString(),
|
|
716
|
+
},
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
}).catch(() => {});
|
|
720
|
+
if (sendStatus !== "not-submitted") {
|
|
721
|
+
error.completionUnknown = true;
|
|
722
|
+
}
|
|
723
|
+
throw error;
|
|
724
|
+
} finally {
|
|
725
|
+
await this.#syncConversationIdentity({ suppressErrors: true });
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
let value;
|
|
729
|
+
try {
|
|
730
|
+
value = await waitForCompletion({
|
|
731
|
+
adapter: this.adapter,
|
|
732
|
+
page: this.adapter.page,
|
|
733
|
+
sendResult,
|
|
734
|
+
timeoutMs,
|
|
735
|
+
});
|
|
736
|
+
const completion = await this.adapter.captureAuxiliaryTurnCompletion?.({
|
|
737
|
+
timeoutMs: Math.min(timeoutMs, 30_000),
|
|
738
|
+
})
|
|
739
|
+
?? null;
|
|
740
|
+
const assistantMessageId = completion?.assistantMessageId
|
|
741
|
+
?? await this.adapter.getLastAssistantMessageId?.()
|
|
742
|
+
?? null;
|
|
743
|
+
const assistantTurn = completion?.assistantTurn
|
|
744
|
+
?? await this.adapter.getLastAssistantTurn?.()
|
|
745
|
+
?? null;
|
|
746
|
+
await this.#queueSessionOperation(() => (
|
|
747
|
+
this.session.completePendingAuxiliaryTurn({
|
|
748
|
+
outboundId,
|
|
749
|
+
assistantMessageId,
|
|
750
|
+
assistantTurn,
|
|
751
|
+
})
|
|
752
|
+
));
|
|
753
|
+
return value;
|
|
754
|
+
} catch (error) {
|
|
755
|
+
// A provider can finish the visible assistant turn yet fail while WTAgent
|
|
756
|
+
// downloads or validates its artifact. Close that browser-turn checkpoint
|
|
757
|
+
// when its final boundary is still provable; otherwise retain it so a
|
|
758
|
+
// later resume cannot blindly submit a duplicate image request.
|
|
759
|
+
try {
|
|
760
|
+
const completion = await this.adapter.captureAuxiliaryTurnCompletion?.({
|
|
761
|
+
timeoutMs: Math.min(timeoutMs, 5_000),
|
|
762
|
+
});
|
|
763
|
+
if (completion) {
|
|
764
|
+
await this.#queueSessionOperation(() => (
|
|
765
|
+
this.session.completePendingAuxiliaryTurn({
|
|
766
|
+
outboundId,
|
|
767
|
+
assistantMessageId: completion.assistantMessageId ?? null,
|
|
768
|
+
assistantTurn: completion.assistantTurn ?? null,
|
|
769
|
+
})
|
|
770
|
+
));
|
|
771
|
+
}
|
|
772
|
+
} catch {
|
|
773
|
+
// Keep pendingAuxiliaryTurn intact when completion cannot be proven.
|
|
774
|
+
}
|
|
775
|
+
// If the completion callback produced a temporary artifact but the
|
|
776
|
+
// assistant-boundary checkpoint failed, ownership never reaches the
|
|
777
|
+
// ImageGenerationService, so clean it here.
|
|
778
|
+
await value?.cleanup?.().catch(() => null);
|
|
779
|
+
error.completionUnknown = true;
|
|
780
|
+
throw error;
|
|
781
|
+
} finally {
|
|
782
|
+
await this.#syncConversationIdentity({ suppressErrors: true });
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
async #appendHandoffTranscript(activeHandoff, suffix, item) {
|
|
787
|
+
if (!activeHandoff?.handoffId) {
|
|
788
|
+
throw new Error("Cannot append transcript without an assistant handoff ID.");
|
|
789
|
+
}
|
|
790
|
+
return await this.session.appendTranscriptItemOnce(item, {
|
|
791
|
+
idempotencyKey: `${activeHandoff.handoffId}:${suffix}`,
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
#inferPendingRuntimeTurn(pendingOutbound) {
|
|
796
|
+
if (Number.isSafeInteger(pendingOutbound?.runtimeTurn)) {
|
|
797
|
+
return pendingOutbound.runtimeTurn;
|
|
798
|
+
}
|
|
799
|
+
const currentTurn = Number(this.session.state.turn || 0);
|
|
800
|
+
if ([
|
|
801
|
+
"empty_response_recovery",
|
|
802
|
+
"dead_request_recovery",
|
|
803
|
+
"generation_failed_recovery",
|
|
804
|
+
].includes(pendingOutbound?.kind)) {
|
|
805
|
+
return Math.max(1, currentTurn);
|
|
806
|
+
}
|
|
807
|
+
return currentTurn + 1;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
#recoveryHandoffPatch(outcome) {
|
|
811
|
+
return {
|
|
812
|
+
conversationUrl: outcome.conversationUrl,
|
|
813
|
+
conversationTargetId: outcome.conversationTargetId,
|
|
814
|
+
userMessageId: outcome.userMessageId,
|
|
815
|
+
userTurn: outcome.userTurn,
|
|
816
|
+
preOutboundMarkerIds: outcome.preOutboundMarkerIds ?? [],
|
|
817
|
+
assistantBaseline: outcome.assistantBaseline ?? null,
|
|
818
|
+
assistantCandidateMessageId:
|
|
819
|
+
outcome.assistantCandidateMessageId ?? null,
|
|
820
|
+
assistantCandidateTurn: outcome.assistantCandidateTurn ?? null,
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
async #refreshHandoffAfterRestore(handoff, restoration) {
|
|
825
|
+
const conversationUrl = restoration?.conversationUrl
|
|
826
|
+
?? this.session.state.conversationUrl
|
|
827
|
+
?? handoff.conversationUrl;
|
|
828
|
+
const conversationTargetId = restoration?.targetId
|
|
829
|
+
?? this.session.state.conversationTargetId
|
|
830
|
+
?? handoff.conversationTargetId;
|
|
831
|
+
const refreshed = await this.#queueSessionOperation(() => (
|
|
832
|
+
this.session.refreshPendingAssistantTurn(handoff.handoffId, {
|
|
833
|
+
conversationUrl,
|
|
834
|
+
conversationTargetId,
|
|
835
|
+
})
|
|
836
|
+
));
|
|
837
|
+
this.adapter.acceptRecoveredOutboundAnchor?.({
|
|
838
|
+
conversationUrl,
|
|
839
|
+
conversationTargetId,
|
|
840
|
+
userMessageId: refreshed.userMessageId,
|
|
841
|
+
userTurn: refreshed.userTurn,
|
|
842
|
+
assistantBaseline: refreshed.assistantBaseline,
|
|
843
|
+
assistantCandidateMessageId:
|
|
844
|
+
refreshed.assistantCandidateMessageId ?? null,
|
|
845
|
+
assistantCandidateTurn: refreshed.assistantCandidateTurn ?? null,
|
|
232
846
|
});
|
|
847
|
+
return refreshed;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
async #recoverPendingConversation({ pendingOutbound, pendingAssistantTurn }) {
|
|
851
|
+
const outboundId = pendingOutbound?.outboundId
|
|
852
|
+
?? pendingAssistantTurn?.sourceOutboundId
|
|
853
|
+
?? null;
|
|
854
|
+
if (!isValidOutboundCorrelationId(outboundId)) {
|
|
855
|
+
throw new BrowserAdapterError(
|
|
856
|
+
"The pending browser handoff has no valid outbound correlation ID.",
|
|
857
|
+
{ code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
|
|
858
|
+
);
|
|
859
|
+
}
|
|
860
|
+
if (!this.adapter.supportsPendingOutboundRecovery?.()) {
|
|
861
|
+
throw new BrowserAdapterError(
|
|
862
|
+
`${this.adapter.providerName ?? "Web provider"} does not support safe pending outbound recovery.`,
|
|
863
|
+
{ code: "OUTBOUND_RECOVERY_UNSUPPORTED", recoverable: false },
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
const boundTargetId = pendingOutbound?.conversationTargetId
|
|
867
|
+
?? (!pendingOutbound
|
|
868
|
+
? pendingAssistantTurn?.conversationTargetId ?? null
|
|
869
|
+
: null);
|
|
870
|
+
const rootTargetId = this.session.state.conversationTargetId ?? null;
|
|
871
|
+
if (
|
|
872
|
+
boundTargetId
|
|
873
|
+
&& rootTargetId
|
|
874
|
+
&& boundTargetId !== rootTargetId
|
|
875
|
+
) {
|
|
876
|
+
throw new BrowserAdapterError(
|
|
877
|
+
"The pending browser handoff target conflicts with the Session target. Refusing to choose either target.",
|
|
878
|
+
{ code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
|
|
879
|
+
);
|
|
880
|
+
}
|
|
881
|
+
const targetId = boundTargetId ?? rootTargetId;
|
|
882
|
+
if (!targetId) {
|
|
883
|
+
throw new BrowserAdapterError(
|
|
884
|
+
"Pending outbound recovery requires the exact saved browser target.",
|
|
885
|
+
{ code: "RECOVERY_TARGET_UNAVAILABLE", recoverable: false },
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
await this.adapter.launchRecoveryTarget(targetId);
|
|
890
|
+
await this.emit("browser.started", {
|
|
891
|
+
profileDir: this.adapter.profileDir,
|
|
892
|
+
recovery: true,
|
|
893
|
+
targetId,
|
|
894
|
+
});
|
|
895
|
+
const priorForReconciliation = pendingOutbound
|
|
896
|
+
&& pendingAssistantTurn?.sourceOutboundId !== pendingOutbound.outboundId
|
|
897
|
+
? null
|
|
898
|
+
: pendingAssistantTurn;
|
|
899
|
+
const outcome = await this.adapter.reconcilePendingOutbound({
|
|
900
|
+
pendingOutbound,
|
|
901
|
+
priorHandoff: priorForReconciliation,
|
|
902
|
+
conversationTargetId: targetId,
|
|
903
|
+
lastUserMessageId: this.session.state.lastUserMessageId ?? null,
|
|
904
|
+
lastAssistantMessageId:
|
|
905
|
+
this.session.state.lastAssistantMessageId ?? null,
|
|
906
|
+
});
|
|
907
|
+
if (!["waiting", "complete"].includes(outcome?.status)) {
|
|
908
|
+
throw new BrowserAdapterError(
|
|
909
|
+
"Pending outbound recovery did not return a durable assistant boundary.",
|
|
910
|
+
{ code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
|
|
911
|
+
);
|
|
912
|
+
}
|
|
913
|
+
const outcomeKind = outcome.conversationUrl
|
|
914
|
+
? this.adapter.classifyConversationUrl?.(outcome.conversationUrl)
|
|
915
|
+
: null;
|
|
916
|
+
if (
|
|
917
|
+
outcome.conversationTargetId !== targetId
|
|
918
|
+
|| !["restorable", "provisional"].includes(outcomeKind)
|
|
919
|
+
) {
|
|
920
|
+
throw new BrowserAdapterError(
|
|
921
|
+
"Pending outbound recovery returned a different browser target or conversation route.",
|
|
922
|
+
{ code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
|
|
923
|
+
);
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
let handoff = pendingAssistantTurn;
|
|
927
|
+
if (pendingOutbound) {
|
|
928
|
+
const pendingToolAcknowledgement =
|
|
929
|
+
pendingOutbound.pendingToolAcknowledgement
|
|
930
|
+
?? (["tool_result", "pending_tool_result"].includes(pendingOutbound.kind)
|
|
931
|
+
&& this.session.state.pendingToolResult
|
|
932
|
+
? {
|
|
933
|
+
callId: this.session.state.pendingToolResult.callId ?? null,
|
|
934
|
+
name: this.session.state.pendingToolResult.name ?? null,
|
|
935
|
+
operationSignature:
|
|
936
|
+
this.session.state.pendingToolResult.operationSignature ?? null,
|
|
937
|
+
}
|
|
938
|
+
: null);
|
|
939
|
+
handoff = await this.#queueSessionOperation(() => (
|
|
940
|
+
this.session.commitPendingOutboundHandoff({
|
|
941
|
+
outboundId: pendingOutbound.outboundId,
|
|
942
|
+
handoff: {
|
|
943
|
+
...this.#recoveryHandoffPatch(outcome),
|
|
944
|
+
runtimeTurn: this.#inferPendingRuntimeTurn(pendingOutbound),
|
|
945
|
+
pendingToolAcknowledgement,
|
|
946
|
+
},
|
|
947
|
+
})
|
|
948
|
+
));
|
|
949
|
+
await this.emit("outbound.reconciled", {
|
|
950
|
+
outboundId: pendingOutbound.outboundId,
|
|
951
|
+
kind: pendingOutbound.kind ?? null,
|
|
952
|
+
status: outcome.status,
|
|
953
|
+
userMessageId: outcome.userMessageId ?? null,
|
|
954
|
+
});
|
|
955
|
+
} else if (handoff) {
|
|
956
|
+
handoff = await this.#queueSessionOperation(() => (
|
|
957
|
+
this.session.refreshPendingAssistantTurn(
|
|
958
|
+
handoff.handoffId,
|
|
959
|
+
this.#recoveryHandoffPatch(outcome),
|
|
960
|
+
)
|
|
961
|
+
));
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
// Recovery reconciles text/identity. A not-yet-checkpointed native turn
|
|
965
|
+
// still needs the ordinary image-aware completion reader before sealing it.
|
|
966
|
+
if (outcome.status === "complete" && ((!this.nativeImageReceiver && !this.nativeMusicReceiver) || handoff.status === "complete")) {
|
|
967
|
+
const outcomeHash = createHash("sha256")
|
|
968
|
+
.update(outcome.rawResponse)
|
|
969
|
+
.digest("hex");
|
|
970
|
+
if (outcome.responseHash && outcome.responseHash !== outcomeHash) {
|
|
971
|
+
throw new BrowserAdapterError(
|
|
972
|
+
"Recovered assistant response hash does not match its raw content.",
|
|
973
|
+
{ code: "OUTBOUND_COMMIT_UNCERTAIN", recoverable: false },
|
|
974
|
+
);
|
|
975
|
+
}
|
|
976
|
+
handoff = await this.#queueSessionOperation(() => (
|
|
977
|
+
this.session.completePendingAssistantTurn({
|
|
978
|
+
handoffId: handoff.handoffId,
|
|
979
|
+
assistantMessageId: outcome.assistantMessageId ?? null,
|
|
980
|
+
assistantTurn: outcome.assistantTurn ?? null,
|
|
981
|
+
rawResponse: outcome.rawResponse,
|
|
982
|
+
nativeImages: handoff.nativeImages ?? [],
|
|
983
|
+
nativeMusic: handoff.nativeMusic ?? [],
|
|
984
|
+
})
|
|
985
|
+
));
|
|
986
|
+
}
|
|
987
|
+
return handoff;
|
|
233
988
|
}
|
|
234
989
|
|
|
235
990
|
async run({
|
|
@@ -243,8 +998,65 @@ export class AgentRuntime {
|
|
|
243
998
|
projectRoot,
|
|
244
999
|
} = this.session.state;
|
|
245
1000
|
const previousConversationUrl = this.session.state.conversationUrl;
|
|
246
|
-
|
|
247
|
-
const
|
|
1001
|
+
const pendingOutboundAtStart = this.session.state.pendingOutbound;
|
|
1002
|
+
const pendingAssistantAtStart = this.session.state.pendingAssistantTurn;
|
|
1003
|
+
const pendingAuxiliaryAtStart = this.session.state.pendingAuxiliaryTurn;
|
|
1004
|
+
if (pendingAuxiliaryAtStart != null) {
|
|
1005
|
+
throw new BrowserAdapterError(
|
|
1006
|
+
"An image/provider auxiliary turn was interrupted after submission. "
|
|
1007
|
+
+ "WTAgent will not replay it automatically because that could create a duplicate remote artifact.",
|
|
1008
|
+
{ code: "AUXILIARY_TURN_RECOVERY_REQUIRED", recoverable: false },
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
const hasPendingRecovery = pendingOutboundAtStart != null
|
|
1012
|
+
|| pendingAssistantAtStart != null;
|
|
1013
|
+
if (this.nativeMusicReceiver && resume && !hasPendingRecovery
|
|
1014
|
+
&& !instruction?.trim() && files.length === 0 && this.session.state.lastMusicResult) {
|
|
1015
|
+
// Bare resume of a completed generation must not spend quota again.
|
|
1016
|
+
const result = this.session.state.lastMusicResult;
|
|
1017
|
+
await this.emit("run.completed", result);
|
|
1018
|
+
return { sessionId: this.session.sessionId, ...result };
|
|
1019
|
+
}
|
|
1020
|
+
const supportsStrictRecovery = Boolean(
|
|
1021
|
+
this.adapter.supportsPendingOutboundRecovery?.(),
|
|
1022
|
+
);
|
|
1023
|
+
const strictRecoveryAtStart = pendingOutboundAtStart != null
|
|
1024
|
+
|| (pendingAssistantAtStart != null && supportsStrictRecovery);
|
|
1025
|
+
const legacyAssistantRecoveryAtStart = pendingOutboundAtStart == null
|
|
1026
|
+
&& pendingAssistantAtStart != null
|
|
1027
|
+
&& !supportsStrictRecovery;
|
|
1028
|
+
if (
|
|
1029
|
+
hasPendingRecovery
|
|
1030
|
+
&& (
|
|
1031
|
+
!resume
|
|
1032
|
+
|| Boolean(instruction?.trim())
|
|
1033
|
+
|| (files ?? []).length > 0
|
|
1034
|
+
)
|
|
1035
|
+
) {
|
|
1036
|
+
throw new BrowserAdapterError(
|
|
1037
|
+
"A pending browser handoff can only be recovered with a bare resume. "
|
|
1038
|
+
+ "Do not add an instruction or attachment until recovery completes.",
|
|
1039
|
+
{ code: "RECOVERY_REQUIRES_BARE_RESUME", recoverable: false },
|
|
1040
|
+
);
|
|
1041
|
+
}
|
|
1042
|
+
const attachments = (files ?? []).map((file) => ({
|
|
1043
|
+
name: file.name ?? null,
|
|
1044
|
+
path: file.path ?? null,
|
|
1045
|
+
}));
|
|
1046
|
+
const messageOptions = attachments.length > 0 ? { attachments } : {};
|
|
1047
|
+
|
|
1048
|
+
// Persist the opening user intent (including attachment metadata) before any
|
|
1049
|
+
// browser work. A later recovery can then distinguish a replayable untouched
|
|
1050
|
+
// task from an unknowable empty legacy rollout.
|
|
1051
|
+
if (!resume) {
|
|
1052
|
+
const transcript = await this.session.readTranscript();
|
|
1053
|
+
if (transcript.items.length === 0) {
|
|
1054
|
+
await this.session.appendTranscriptItemOnce(
|
|
1055
|
+
userMessage(task, messageOptions),
|
|
1056
|
+
{ idempotencyKey: `session:${this.session.sessionId}:opening` },
|
|
1057
|
+
);
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
248
1060
|
|
|
249
1061
|
await this.session.update({
|
|
250
1062
|
phase: "initializing",
|
|
@@ -253,10 +1065,49 @@ export class AgentRuntime {
|
|
|
253
1065
|
});
|
|
254
1066
|
await this.emit("runtime.initializing");
|
|
255
1067
|
|
|
1068
|
+
const modelTurnTimeoutMs = this.limits.modelTurnTimeoutMs;
|
|
1069
|
+
let pendingToolResult;
|
|
1070
|
+
let awaitingPendingAcknowledgement = false;
|
|
1071
|
+
let replayGuard = null;
|
|
1072
|
+
let protocolErrors = 0;
|
|
1073
|
+
const baseTurn = resume ? Number(this.session.state.turn || 0) : 0;
|
|
1074
|
+
|
|
1075
|
+
if (strictRecoveryAtStart) {
|
|
1076
|
+
const recoveredHandoff = await this.#recoverPendingConversation({
|
|
1077
|
+
pendingOutbound: pendingOutboundAtStart,
|
|
1078
|
+
pendingAssistantTurn: pendingAssistantAtStart,
|
|
1079
|
+
});
|
|
1080
|
+
// Recovery proves the browser boundary before touching a possibly-running
|
|
1081
|
+
// side-effect ledger. The existing local exactly-once rules then apply.
|
|
1082
|
+
await this.session.recoverInterruptedSideEffects();
|
|
1083
|
+
pendingToolResult = this.session.state.pendingToolResult;
|
|
1084
|
+
awaitingPendingAcknowledgement = Boolean(
|
|
1085
|
+
recoveredHandoff.pendingToolAcknowledgement,
|
|
1086
|
+
);
|
|
1087
|
+
replayGuard = pendingToolResult?.operationSignature
|
|
1088
|
+
? {
|
|
1089
|
+
signature: pendingToolResult.operationSignature,
|
|
1090
|
+
result: pendingToolResult,
|
|
1091
|
+
}
|
|
1092
|
+
: null;
|
|
1093
|
+
await this.session.update({
|
|
1094
|
+
phase: "running",
|
|
1095
|
+
activeMode: null,
|
|
1096
|
+
});
|
|
1097
|
+
await this.emit("conversation.started", {
|
|
1098
|
+
url: recoveredHandoff.conversationUrl,
|
|
1099
|
+
recovered: true,
|
|
1100
|
+
});
|
|
1101
|
+
} else {
|
|
256
1102
|
// On resume, prefer an existing tab already showing the conversation so
|
|
257
1103
|
// repeated resumes reuse the same tab instead of piling up new ones.
|
|
258
1104
|
await this.adapter.launch(
|
|
259
1105
|
resume ? this.session.state.conversationUrl : null,
|
|
1106
|
+
{
|
|
1107
|
+
preferredTargetId: resume
|
|
1108
|
+
? this.session.state.conversationTargetId ?? null
|
|
1109
|
+
: null,
|
|
1110
|
+
},
|
|
260
1111
|
);
|
|
261
1112
|
await this.emit("browser.started", {
|
|
262
1113
|
profileDir: this.adapter.profileDir,
|
|
@@ -288,47 +1139,156 @@ export class AgentRuntime {
|
|
|
288
1139
|
await this.emit("browser.authenticated");
|
|
289
1140
|
}
|
|
290
1141
|
|
|
291
|
-
await this.adapter.startConversation(
|
|
1142
|
+
let restoration = await this.adapter.startConversation(
|
|
292
1143
|
resume ? previousConversationUrl : null,
|
|
293
1144
|
{
|
|
294
1145
|
expectedAssistantMessageId: resume
|
|
295
1146
|
? this.session.state.lastAssistantMessageId
|
|
296
1147
|
: null,
|
|
1148
|
+
expectedUserMessageId: resume
|
|
1149
|
+
? this.session.state.lastUserMessageId
|
|
1150
|
+
: null,
|
|
297
1151
|
},
|
|
298
1152
|
);
|
|
1153
|
+
if (
|
|
1154
|
+
restoration?.status !== "restored-existing"
|
|
1155
|
+
&& restoration?.status !== "verified-fresh"
|
|
1156
|
+
) {
|
|
1157
|
+
throw new BrowserAdapterError(
|
|
1158
|
+
`${this.adapter.providerName} did not report whether conversation history was restored.`,
|
|
1159
|
+
{
|
|
1160
|
+
code: "CONVERSATION_RESTORE_UNVERIFIED",
|
|
1161
|
+
recoverable: false,
|
|
1162
|
+
},
|
|
1163
|
+
);
|
|
1164
|
+
}
|
|
1165
|
+
if (
|
|
1166
|
+
legacyAssistantRecoveryAtStart
|
|
1167
|
+
&& restoration.status !== "restored-existing"
|
|
1168
|
+
) {
|
|
1169
|
+
throw new BrowserAdapterError(
|
|
1170
|
+
`${this.adapter.providerName} could not restore the confirmed pending assistant turn. `
|
|
1171
|
+
+ "Refusing to continue it on a blank conversation.",
|
|
1172
|
+
{
|
|
1173
|
+
code: "CONVERSATION_RESTORE_REQUIRED",
|
|
1174
|
+
recoverable: false,
|
|
1175
|
+
},
|
|
1176
|
+
);
|
|
1177
|
+
}
|
|
1178
|
+
await this.#syncConversationIdentity({
|
|
1179
|
+
identity: {
|
|
1180
|
+
conversationUrl: restoration.conversationUrl,
|
|
1181
|
+
targetId: restoration.targetId ?? null,
|
|
1182
|
+
kind: restoration.status === "verified-fresh"
|
|
1183
|
+
? "fresh"
|
|
1184
|
+
: this.adapter.classifyConversationUrl?.(restoration.conversationUrl),
|
|
1185
|
+
},
|
|
1186
|
+
allowFresh: restoration.status === "verified-fresh",
|
|
1187
|
+
});
|
|
1188
|
+
|
|
1189
|
+
const resumedExistingConversation = resume
|
|
1190
|
+
&& restoration.status === "restored-existing";
|
|
1191
|
+
const rebuildingFreshConversation = resume
|
|
1192
|
+
&& restoration.status === "verified-fresh";
|
|
1193
|
+
let effectiveInstruction = instruction;
|
|
1194
|
+
if (rebuildingFreshConversation) {
|
|
1195
|
+
const {
|
|
1196
|
+
blockers,
|
|
1197
|
+
rebuildInstruction,
|
|
1198
|
+
} = await this.#freshRebuildBlockers({ instruction });
|
|
1199
|
+
if (blockers.length > 0) {
|
|
1200
|
+
throw new BrowserAdapterError(
|
|
1201
|
+
`${this.adapter.providerName} opened an empty conversation, but this session has `
|
|
1202
|
+
+ `history that cannot be replayed safely (${blockers.join(", ")}). `
|
|
1203
|
+
+ "The browser message was not sent.",
|
|
1204
|
+
{
|
|
1205
|
+
code: "CONVERSATION_REBUILD_UNSAFE",
|
|
1206
|
+
recoverable: false,
|
|
1207
|
+
details: { blockers },
|
|
1208
|
+
},
|
|
1209
|
+
);
|
|
1210
|
+
}
|
|
1211
|
+
effectiveInstruction = rebuildInstruction;
|
|
1212
|
+
await this.emit("conversation.rebuilding_fresh", {
|
|
1213
|
+
previousUrl: previousConversationUrl,
|
|
1214
|
+
url: restoration.conversationUrl,
|
|
1215
|
+
});
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// Only normalize an interrupted side-effect ledger after the browser
|
|
1219
|
+
// restoration decision. An unsafe fresh-page fallback must leave every
|
|
1220
|
+
// pending/result/side-effect record untouched for deliberate inspection.
|
|
1221
|
+
await this.session.recoverInterruptedSideEffects();
|
|
1222
|
+
pendingToolResult = this.session.state.pendingToolResult;
|
|
299
1223
|
|
|
300
1224
|
// Optional setup after authentication and the target conversation are
|
|
301
1225
|
// ready, but before the first message is sent. Interactive CLI runs use
|
|
302
1226
|
// this to let the user choose a model directly on the provider website.
|
|
303
1227
|
// WTAgent never names, selects, or overrides a provider-specific model.
|
|
304
|
-
//
|
|
305
|
-
|
|
1228
|
+
// Existing conversations keep their model context. A verified fresh rebuild
|
|
1229
|
+
// is a genuinely new web chat, so interactive runs offer the same browser-side
|
|
1230
|
+
// model choice as an ordinary bootstrap.
|
|
1231
|
+
if (restoration.status === "verified-fresh" && this.postAuthSetup) {
|
|
306
1232
|
await this.postAuthSetup({ adapter: this.adapter });
|
|
1233
|
+
// Browser-side setup is user-controlled and can navigate the tab. Reopen
|
|
1234
|
+
// and re-verify an empty chat after it returns so the bootstrap can never
|
|
1235
|
+
// land in a history conversation selected during setup.
|
|
1236
|
+
restoration = await this.adapter.startConversation();
|
|
1237
|
+
if (restoration?.status !== "verified-fresh") {
|
|
1238
|
+
throw new BrowserAdapterError(
|
|
1239
|
+
`${this.adapter.providerName} did not remain on a verified empty conversation after browser setup.`,
|
|
1240
|
+
{
|
|
1241
|
+
code: "CONVERSATION_NOT_FRESH",
|
|
1242
|
+
recoverable: false,
|
|
1243
|
+
},
|
|
1244
|
+
);
|
|
1245
|
+
}
|
|
1246
|
+
await this.#syncConversationIdentity({
|
|
1247
|
+
identity: {
|
|
1248
|
+
conversationUrl: restoration.conversationUrl,
|
|
1249
|
+
targetId: restoration.targetId ?? null,
|
|
1250
|
+
kind: "fresh",
|
|
1251
|
+
},
|
|
1252
|
+
allowFresh: true,
|
|
1253
|
+
});
|
|
307
1254
|
}
|
|
308
1255
|
|
|
309
|
-
const modelTurnTimeoutMs = this.limits.modelTurnTimeoutMs;
|
|
310
|
-
|
|
311
1256
|
await this.session.update({
|
|
312
1257
|
phase: "running",
|
|
313
|
-
conversationUrl: await this.adapter.getConversationUrl(),
|
|
314
1258
|
activeMode: null,
|
|
315
1259
|
});
|
|
316
1260
|
await this.emit("conversation.started", {
|
|
317
1261
|
url: this.session.state.conversationUrl,
|
|
318
1262
|
});
|
|
319
1263
|
|
|
1264
|
+
if (legacyAssistantRecoveryAtStart) {
|
|
1265
|
+
const recoveredHandoff = await this.#refreshHandoffAfterRestore(
|
|
1266
|
+
pendingAssistantAtStart,
|
|
1267
|
+
restoration,
|
|
1268
|
+
);
|
|
1269
|
+
awaitingPendingAcknowledgement = Boolean(
|
|
1270
|
+
recoveredHandoff.pendingToolAcknowledgement,
|
|
1271
|
+
);
|
|
1272
|
+
replayGuard = pendingToolResult?.operationSignature
|
|
1273
|
+
? {
|
|
1274
|
+
signature: pendingToolResult.operationSignature,
|
|
1275
|
+
result: pendingToolResult,
|
|
1276
|
+
}
|
|
1277
|
+
: null;
|
|
1278
|
+
}
|
|
320
1279
|
let initialMessage;
|
|
321
1280
|
let initialKind;
|
|
322
1281
|
// The web transport gets `.web` (XML/marked text). The portable rollout
|
|
323
|
-
// records only
|
|
1282
|
+
// records only confirmed user messages; WTAgent scaffolding stays transport-only.
|
|
324
1283
|
let initialTranscript = [];
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
1284
|
+
if (!legacyAssistantRecoveryAtStart) {
|
|
1285
|
+
if (this.nativeMusicReceiver) {
|
|
1286
|
+
if (inPlaceRecovery) throw new Error("Music generation is not automatically repeated; resume the pending turn without a new instruction.");
|
|
1287
|
+
const request = effectiveInstruction?.trim() || task;
|
|
1288
|
+
initialMessage = `Create the following music using the native Music tool.\n\n${request}`;
|
|
1289
|
+
initialTranscript = resume && effectiveInstruction?.trim() ? [userMessage(effectiveInstruction.trim(), messageOptions)] : [];
|
|
1290
|
+
initialKind = resume ? "music_follow_up" : "music_generation";
|
|
1291
|
+
} else if (resumedExistingConversation && pendingToolResult && !inPlaceRecovery) {
|
|
332
1292
|
let suffix = "";
|
|
333
1293
|
if (instruction?.trim()) {
|
|
334
1294
|
suffix = `\n<resume_instruction>${cdata(instruction)}</resume_instruction>`;
|
|
@@ -336,7 +1296,7 @@ export class AgentRuntime {
|
|
|
336
1296
|
}
|
|
337
1297
|
initialMessage = this.buildToolResultMessage(pendingToolResult, { suffix });
|
|
338
1298
|
initialKind = "pending_tool_result";
|
|
339
|
-
} else if (
|
|
1299
|
+
} else if (resumedExistingConversation && instruction?.trim()) {
|
|
340
1300
|
// The live web conversation already contains the bootstrap protocol
|
|
341
1301
|
// and tool catalog. A normal follow-up should be the user's message, not
|
|
342
1302
|
// another several-thousand-character protocol bootstrap. sendMessage()
|
|
@@ -344,7 +1304,7 @@ export class AgentRuntime {
|
|
|
344
1304
|
initialMessage = instruction.trim();
|
|
345
1305
|
initialTranscript = [userMessage(instruction.trim(), messageOptions)];
|
|
346
1306
|
initialKind = "follow_up";
|
|
347
|
-
} else if (
|
|
1307
|
+
} else if (resumedExistingConversation && inPlaceRecovery) {
|
|
348
1308
|
// The original request/tool result is already visible in this live web
|
|
349
1309
|
// conversation. Ask the provider to continue without duplicating transport
|
|
350
1310
|
// payloads, attachments, or canonical transcript entries.
|
|
@@ -352,67 +1312,127 @@ export class AgentRuntime {
|
|
|
352
1312
|
initialKind = "empty_response_recovery";
|
|
353
1313
|
} else if (resume) {
|
|
354
1314
|
const prompt = buildResumePrompt({
|
|
355
|
-
instruction
|
|
1315
|
+
instruction: rebuildingFreshConversation
|
|
1316
|
+
? effectiveInstruction
|
|
1317
|
+
: instruction,
|
|
356
1318
|
state: this.session.state,
|
|
357
1319
|
tools: this.registry.list(),
|
|
1320
|
+
nativeImages: Boolean(this.nativeImageReceiver),
|
|
358
1321
|
});
|
|
359
1322
|
initialMessage = prompt.web;
|
|
360
|
-
|
|
361
|
-
|
|
1323
|
+
// The original task is already the one safe canonical item required by the
|
|
1324
|
+
// rebuild gate. Record only a newly confirmed follow-up instruction.
|
|
1325
|
+
initialTranscript = rebuildingFreshConversation
|
|
1326
|
+
? (effectiveInstruction?.trim()
|
|
1327
|
+
? [userMessage(effectiveInstruction.trim(), messageOptions)]
|
|
1328
|
+
: [])
|
|
1329
|
+
: [userMessage(prompt.user, messageOptions)];
|
|
1330
|
+
initialKind = rebuildingFreshConversation ? "fresh_rebuild" : "resume";
|
|
362
1331
|
} else {
|
|
363
1332
|
const prompt = buildBootstrapPrompt({
|
|
364
1333
|
task,
|
|
365
1334
|
projectRoot,
|
|
366
1335
|
tools: this.registry.list(),
|
|
1336
|
+
nativeImages: Boolean(this.nativeImageReceiver),
|
|
367
1337
|
});
|
|
368
1338
|
initialMessage = prompt.web;
|
|
369
|
-
|
|
1339
|
+
// The opening item was persisted before browser setup above.
|
|
1340
|
+
initialTranscript = [];
|
|
370
1341
|
initialKind = "bootstrap";
|
|
371
1342
|
}
|
|
372
|
-
|
|
373
|
-
for (const item of initialTranscript) {
|
|
374
|
-
await this.session.appendTranscriptItem(item);
|
|
375
1343
|
}
|
|
376
|
-
await this.sendMessage(initialMessage, {
|
|
377
|
-
files,
|
|
378
|
-
maxBytes: initialKind === "pending_tool_result"
|
|
379
|
-
? this.limits.maxBrowserToolResultBytes
|
|
380
|
-
: null,
|
|
381
|
-
});
|
|
382
|
-
let awaitingPendingAcknowledgement = Boolean(pendingToolResult);
|
|
383
|
-
let replayGuard = pendingToolResult?.operationSignature
|
|
384
|
-
? {
|
|
385
|
-
signature: pendingToolResult.operationSignature,
|
|
386
|
-
result: pendingToolResult,
|
|
387
|
-
}
|
|
388
|
-
: null;
|
|
389
|
-
await this.emit("model.message_sent", { kind: initialKind });
|
|
390
1344
|
|
|
391
|
-
|
|
392
|
-
|
|
1345
|
+
if (!legacyAssistantRecoveryAtStart) {
|
|
1346
|
+
await this.sendMessage(initialMessage, {
|
|
1347
|
+
files,
|
|
1348
|
+
maxBytes: initialKind === "pending_tool_result"
|
|
1349
|
+
? this.limits.maxBrowserToolResultBytes
|
|
1350
|
+
: null,
|
|
1351
|
+
// A full bootstrap/rebuild prompt may reconnect only when submission was
|
|
1352
|
+
// proven not to have started. Context-dependent messages never opt in.
|
|
1353
|
+
allowFreshReconnect: restoration.status === "verified-fresh",
|
|
1354
|
+
allowAssistantContinuation: initialKind === "empty_response_recovery",
|
|
1355
|
+
outboundKind: initialKind,
|
|
1356
|
+
transcriptItems: initialTranscript,
|
|
1357
|
+
runtimeTurn: baseTurn + 1,
|
|
1358
|
+
pendingToolAcknowledgement: pendingToolResult
|
|
1359
|
+
? {
|
|
1360
|
+
callId: pendingToolResult.callId ?? null,
|
|
1361
|
+
name: pendingToolResult.name ?? null,
|
|
1362
|
+
operationSignature: pendingToolResult.operationSignature ?? null,
|
|
1363
|
+
}
|
|
1364
|
+
: null,
|
|
1365
|
+
});
|
|
1366
|
+
awaitingPendingAcknowledgement = Boolean(pendingToolResult);
|
|
1367
|
+
replayGuard = pendingToolResult?.operationSignature
|
|
1368
|
+
? {
|
|
1369
|
+
signature: pendingToolResult.operationSignature,
|
|
1370
|
+
result: pendingToolResult,
|
|
1371
|
+
}
|
|
1372
|
+
: null;
|
|
1373
|
+
await this.emit("model.message_sent", { kind: initialKind });
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
393
1376
|
|
|
394
1377
|
for (let step = 1; ; step += 1) {
|
|
395
|
-
|
|
1378
|
+
let activeHandoff = this.session.state.pendingAssistantTurn;
|
|
1379
|
+
if (!activeHandoff) {
|
|
1380
|
+
throw new BrowserAdapterError(
|
|
1381
|
+
"The durable assistant handoff disappeared before response processing.",
|
|
1382
|
+
{ code: "ASSISTANT_HANDOFF_MISSING", recoverable: false },
|
|
1383
|
+
);
|
|
1384
|
+
}
|
|
1385
|
+
const turnNumber = Number.isSafeInteger(activeHandoff.runtimeTurn)
|
|
1386
|
+
? activeHandoff.runtimeTurn
|
|
1387
|
+
: baseTurn + step;
|
|
396
1388
|
await this.session.update({ turn: turnNumber, phase: "waiting_model" });
|
|
397
1389
|
let raw;
|
|
1390
|
+
let nativeImages = [];
|
|
1391
|
+
let nativeMusic = [];
|
|
398
1392
|
let emptyAssistantRetries = 0;
|
|
399
1393
|
let connectionRetries = 0;
|
|
400
1394
|
for (;;) {
|
|
401
1395
|
try {
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
await this.
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
1396
|
+
if (activeHandoff.status === "complete") {
|
|
1397
|
+
raw = activeHandoff.rawResponse;
|
|
1398
|
+
nativeImages = activeHandoff.nativeImages ?? [];
|
|
1399
|
+
nativeMusic = activeHandoff.nativeMusic ?? [];
|
|
1400
|
+
} else {
|
|
1401
|
+
try {
|
|
1402
|
+
raw = await this.adapter.waitForTurnComplete({
|
|
1403
|
+
timeoutMs: modelTurnTimeoutMs,
|
|
1404
|
+
stableWindowMs: this.limits.modelStableWindowMs,
|
|
1405
|
+
emptyResponseWindowMs: this.limits.emptyAssistantWindowMs,
|
|
1406
|
+
deadRequestGraceMs: this.limits.deadRequestGraceMs,
|
|
1407
|
+
readNativeImages: this.nativeImageReceiver
|
|
1408
|
+
? (message) => this.nativeImageReceiver.read(message) : null,
|
|
1409
|
+
readNativeMusic: this.nativeMusicReceiver
|
|
1410
|
+
? (message) => this.nativeMusicReceiver.read(message) : null,
|
|
1411
|
+
onDelta: async (delta) => {
|
|
1412
|
+
await this.onEvent?.({
|
|
1413
|
+
type: "model.streaming",
|
|
1414
|
+
sessionId: this.session.sessionId,
|
|
1415
|
+
timestamp: new Date().toISOString(),
|
|
1416
|
+
payload: { delta },
|
|
1417
|
+
});
|
|
1418
|
+
},
|
|
413
1419
|
});
|
|
414
|
-
|
|
415
|
-
|
|
1420
|
+
if (raw && typeof raw === "object" && Array.isArray(raw.nativeImages)) {
|
|
1421
|
+
nativeImages = raw.nativeImages;
|
|
1422
|
+
raw = raw.text ?? "";
|
|
1423
|
+
} else if (raw && typeof raw === "object" && Array.isArray(raw.nativeMusic)) {
|
|
1424
|
+
nativeMusic = raw.nativeMusic;
|
|
1425
|
+
raw = raw.text ?? "";
|
|
1426
|
+
}
|
|
1427
|
+
} finally {
|
|
1428
|
+
// Navigation events normally persist canonicalization immediately;
|
|
1429
|
+
// this validated fallback runs on every outcome but never masks a
|
|
1430
|
+
// completed reply or the original browser/model error.
|
|
1431
|
+
await this.#syncConversationIdentity({
|
|
1432
|
+
suppressErrors: true,
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
416
1436
|
break;
|
|
417
1437
|
} catch (error) {
|
|
418
1438
|
if (isConnectionLostError(error)) {
|
|
@@ -420,9 +1440,25 @@ export class AgentRuntime {
|
|
|
420
1440
|
throw error;
|
|
421
1441
|
}
|
|
422
1442
|
connectionRetries += 1;
|
|
423
|
-
|
|
1443
|
+
if (supportsStrictRecovery) {
|
|
1444
|
+
await this.adapter.detach?.();
|
|
1445
|
+
activeHandoff = await this.#recoverPendingConversation({
|
|
1446
|
+
pendingOutbound: null,
|
|
1447
|
+
pendingAssistantTurn: activeHandoff,
|
|
1448
|
+
});
|
|
1449
|
+
} else {
|
|
1450
|
+
const { checkpoint } = await this.#reconnectAndRestore({
|
|
1451
|
+
checkpointIdentity: (restoration) => (
|
|
1452
|
+
this.#refreshHandoffAfterRestore(
|
|
1453
|
+
activeHandoff,
|
|
1454
|
+
restoration,
|
|
1455
|
+
)
|
|
1456
|
+
),
|
|
1457
|
+
});
|
|
1458
|
+
activeHandoff = checkpoint;
|
|
1459
|
+
}
|
|
424
1460
|
// The message was already sent before the connection died; resume
|
|
425
|
-
// waiting for the reply on the restored
|
|
1461
|
+
// waiting for the reply on the restored exact conversation boundary.
|
|
426
1462
|
continue;
|
|
427
1463
|
}
|
|
428
1464
|
|
|
@@ -437,6 +1473,7 @@ export class AgentRuntime {
|
|
|
437
1473
|
|
|
438
1474
|
const deadRequest = error?.code === "DEAD_ASSISTANT_REQUEST";
|
|
439
1475
|
const generationFailed = error?.code === "GENERATION_FAILED";
|
|
1476
|
+
if (this.nativeMusicReceiver) throw error;
|
|
440
1477
|
if (
|
|
441
1478
|
error?.code !== "EMPTY_ASSISTANT_RESPONSE"
|
|
442
1479
|
&& !deadRequest
|
|
@@ -448,7 +1485,6 @@ export class AgentRuntime {
|
|
|
448
1485
|
const emptyAssistantMessageId = await this.adapter
|
|
449
1486
|
.getLastAssistantMessageId?.() ?? null;
|
|
450
1487
|
await this.session.update({
|
|
451
|
-
conversationUrl: await this.adapter.getConversationUrl(),
|
|
452
1488
|
lastAssistantMessageId: emptyAssistantMessageId
|
|
453
1489
|
?? this.session.state.lastAssistantMessageId,
|
|
454
1490
|
});
|
|
@@ -488,13 +1524,25 @@ export class AgentRuntime {
|
|
|
488
1524
|
// Do not resend the original request or tool result: both are already
|
|
489
1525
|
// present in ChatGPT's conversation. This transport-only continuation
|
|
490
1526
|
// also cannot re-execute a local tool by itself.
|
|
491
|
-
await this.sendMessage(
|
|
1527
|
+
const continuationSend = await this.sendMessage(
|
|
492
1528
|
deadRequest
|
|
493
1529
|
? DEAD_REQUEST_CONTINUE_MESSAGE
|
|
494
1530
|
: generationFailed
|
|
495
1531
|
? GENERATION_FAILED_CONTINUE_MESSAGE
|
|
496
1532
|
: EMPTY_ASSISTANT_CONTINUE_MESSAGE,
|
|
1533
|
+
{
|
|
1534
|
+
allowAssistantContinuation: true,
|
|
1535
|
+
outboundKind: deadRequest
|
|
1536
|
+
? "dead_request_recovery"
|
|
1537
|
+
: generationFailed
|
|
1538
|
+
? "generation_failed_recovery"
|
|
1539
|
+
: "empty_response_recovery",
|
|
1540
|
+
runtimeTurn: turnNumber,
|
|
1541
|
+
pendingToolAcknowledgement:
|
|
1542
|
+
activeHandoff.pendingToolAcknowledgement ?? null,
|
|
1543
|
+
},
|
|
497
1544
|
);
|
|
1545
|
+
activeHandoff = continuationSend.pendingAssistantTurn;
|
|
498
1546
|
await this.emit("model.message_sent", {
|
|
499
1547
|
kind: deadRequest
|
|
500
1548
|
? "dead_request_recovery"
|
|
@@ -505,27 +1553,131 @@ export class AgentRuntime {
|
|
|
505
1553
|
});
|
|
506
1554
|
}
|
|
507
1555
|
}
|
|
508
|
-
const assistantMessageId =
|
|
509
|
-
.
|
|
1556
|
+
const assistantMessageId = activeHandoff.status === "complete"
|
|
1557
|
+
? activeHandoff.assistantMessageId ?? null
|
|
1558
|
+
: await this.adapter.getLastAssistantMessageId?.() ?? null;
|
|
1559
|
+
const assistantTurn = activeHandoff.status === "complete"
|
|
1560
|
+
? activeHandoff.assistantTurn ?? null
|
|
1561
|
+
: await this.adapter.getLastAssistantTurn?.() ?? null;
|
|
1562
|
+
if (activeHandoff.status === "waiting") {
|
|
1563
|
+
activeHandoff = await this.#queueSessionOperation(() => (
|
|
1564
|
+
this.session.completePendingAssistantTurn({
|
|
1565
|
+
handoffId: activeHandoff.handoffId,
|
|
1566
|
+
assistantMessageId,
|
|
1567
|
+
assistantTurn,
|
|
1568
|
+
rawResponse: raw,
|
|
1569
|
+
nativeImages,
|
|
1570
|
+
nativeMusic,
|
|
1571
|
+
})
|
|
1572
|
+
));
|
|
1573
|
+
} else {
|
|
1574
|
+
const rawHash = createHash("sha256").update(raw).digest("hex");
|
|
1575
|
+
if (rawHash !== activeHandoff.responseHash) {
|
|
1576
|
+
throw new BrowserAdapterError(
|
|
1577
|
+
"The durable assistant response changed before processing.",
|
|
1578
|
+
{ code: "ASSISTANT_HANDOFF_COLLISION", recoverable: false },
|
|
1579
|
+
);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
510
1582
|
if (awaitingPendingAcknowledgement) {
|
|
511
|
-
await this
|
|
512
|
-
awaitingPendingAcknowledgement = false;
|
|
1583
|
+
await this.#queueSessionUpdate({ pendingToolResult: null });
|
|
513
1584
|
}
|
|
514
|
-
|
|
515
|
-
conversationUrl: await this.adapter.getConversationUrl(),
|
|
516
|
-
// Null is meaningful: retaining an older ID would falsely prove only
|
|
517
|
-
// that stale history had hydrated on the next resume.
|
|
518
|
-
lastAssistantMessageId: assistantMessageId,
|
|
519
|
-
});
|
|
1585
|
+
awaitingPendingAcknowledgement = false;
|
|
520
1586
|
await this.emit("model.message_complete", {
|
|
521
1587
|
turn: turnNumber,
|
|
522
1588
|
raw,
|
|
523
1589
|
assistantMessageId,
|
|
524
1590
|
});
|
|
525
1591
|
|
|
1592
|
+
if (this.nativeMusicReceiver) {
|
|
1593
|
+
let artifacts = activeHandoff.nativeArtifacts;
|
|
1594
|
+
if (!artifacts) {
|
|
1595
|
+
artifacts = nativeMusic.length ? await this.nativeMusicReceiver.save(nativeMusic, {
|
|
1596
|
+
projectRoot, handoffId: activeHandoff.handoffId, assistantMessageId,
|
|
1597
|
+
}) : [];
|
|
1598
|
+
activeHandoff = await this.#queueSessionOperation(() => (
|
|
1599
|
+
this.session.refreshPendingAssistantTurn(activeHandoff.handoffId, { nativeArtifacts: artifacts })
|
|
1600
|
+
));
|
|
1601
|
+
}
|
|
1602
|
+
const message = [
|
|
1603
|
+
raw,
|
|
1604
|
+
...artifacts.map((artifact) => `Saved music: ${artifact.localPath} (${artifact.durationSeconds.toFixed(2)} s)`),
|
|
1605
|
+
...artifacts.map((artifact) => artifact.audioExtractionNote).filter(Boolean),
|
|
1606
|
+
...(artifacts.length ? [] : ["No music file was generated or saved."]),
|
|
1607
|
+
].filter(Boolean).join("\n");
|
|
1608
|
+
await this.#appendHandoffTranscript(activeHandoff, "assistant", assistantMessage(message));
|
|
1609
|
+
const result = { message, artifacts };
|
|
1610
|
+
await this.#queueSessionOperation(() => this.session.clearPendingAssistantTurn(activeHandoff.handoffId, {
|
|
1611
|
+
phase: "idle", lastMessage: message, pendingToolResult: null, lastMusicResult: result,
|
|
1612
|
+
}));
|
|
1613
|
+
if (artifacts.length) await this.emit("model.native_music", { turn: turnNumber, artifacts });
|
|
1614
|
+
await this.emit("run.completed", result);
|
|
1615
|
+
return { sessionId: this.session.sessionId, ...result };
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
if (nativeImages.length > 0) {
|
|
1619
|
+
if (!this.nativeImageReceiver) {
|
|
1620
|
+
throw new Error("This session has pending native images but no image receiver.");
|
|
1621
|
+
}
|
|
1622
|
+
let artifacts = activeHandoff.nativeArtifacts;
|
|
1623
|
+
if (!artifacts) {
|
|
1624
|
+
artifacts = await this.nativeImageReceiver.save(nativeImages, {
|
|
1625
|
+
projectRoot,
|
|
1626
|
+
handoffId: activeHandoff.handoffId,
|
|
1627
|
+
assistantMessageId,
|
|
1628
|
+
});
|
|
1629
|
+
activeHandoff = await this.#queueSessionOperation(() => (
|
|
1630
|
+
this.session.refreshPendingAssistantTurn(activeHandoff.handoffId, { nativeArtifacts: artifacts })
|
|
1631
|
+
));
|
|
1632
|
+
}
|
|
1633
|
+
protocolErrors = 0;
|
|
1634
|
+
const message = [raw, ...artifacts.map((artifact) => `Saved image: ${artifact.localPath}`)]
|
|
1635
|
+
.filter(Boolean).join("\n");
|
|
1636
|
+
await this.#appendHandoffTranscript(activeHandoff, "assistant", assistantMessage(message));
|
|
1637
|
+
await this.emit("model.native_images", { turn: turnNumber, artifacts });
|
|
1638
|
+
await this.emit("model.progress", { turn: turnNumber, message });
|
|
1639
|
+
const receipt = artifacts.map(({ localPath, mimeType, width, height, size, sha256 }) => (
|
|
1640
|
+
{ localPath, mimeType, width, height, size, sha256 }
|
|
1641
|
+
));
|
|
1642
|
+
await this.sendMessage(
|
|
1643
|
+
`<native_image_result>${cdata(JSON.stringify(receipt))}</native_image_result>\n`
|
|
1644
|
+
+ "The images from your preceding reply are saved at these verified local paths. "
|
|
1645
|
+
+ "Continue the user's original task. Generate further images directly if needed; "
|
|
1646
|
+
+ "use local tools for file operations, or finish with a done=true XML answer.",
|
|
1647
|
+
{ outboundKind: "native_image_result", runtimeTurn: turnNumber + 1 },
|
|
1648
|
+
);
|
|
1649
|
+
continue;
|
|
1650
|
+
}
|
|
1651
|
+
|
|
526
1652
|
let parsed;
|
|
1653
|
+
let finalMessage;
|
|
1654
|
+
let usedTrailingProse = false;
|
|
527
1655
|
try {
|
|
528
1656
|
parsed = parseAgentResponse(raw);
|
|
1657
|
+
finalMessage = parsed.message;
|
|
1658
|
+
// Some providers put the deliverable after a done=true envelope. Merge
|
|
1659
|
+
// that display text before validating the final answer.
|
|
1660
|
+
if (parsed.done) {
|
|
1661
|
+
const trailing = extractTrailingProse(raw);
|
|
1662
|
+
if (trailing) {
|
|
1663
|
+
finalMessage = [parsed.message.trim(), trailing]
|
|
1664
|
+
.filter(Boolean)
|
|
1665
|
+
.join("\n\n");
|
|
1666
|
+
usedTrailingProse = true;
|
|
1667
|
+
}
|
|
1668
|
+
if (!finalMessage.trim()) {
|
|
1669
|
+
throw new ProtocolError("done=true requires a non-empty final message.");
|
|
1670
|
+
}
|
|
1671
|
+
if (
|
|
1672
|
+
!parsed.toolCall
|
|
1673
|
+
&& /<tool_calls[\s>]|<tool_call[\s>]|<invoke[\s>]/i.test(finalMessage)
|
|
1674
|
+
) {
|
|
1675
|
+
throw new ProtocolError(
|
|
1676
|
+
"Tool calls must use the <tool_call> element, not the <message> text.",
|
|
1677
|
+
);
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
// Syntax and semantic errors share one consecutive-retry budget.
|
|
529
1681
|
protocolErrors = 0;
|
|
530
1682
|
} catch (error) {
|
|
531
1683
|
if (!(error instanceof ProtocolError)) {
|
|
@@ -546,41 +1698,33 @@ export class AgentRuntime {
|
|
|
546
1698
|
);
|
|
547
1699
|
}
|
|
548
1700
|
|
|
549
|
-
//
|
|
550
|
-
//
|
|
551
|
-
//
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
.
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
});
|
|
568
|
-
await this.session.update({
|
|
569
|
-
phase: "idle",
|
|
570
|
-
lastMessage: plainAnswer,
|
|
571
|
-
pendingToolResult: null,
|
|
572
|
-
});
|
|
573
|
-
await this.session.appendTranscriptItem(assistantMessage(plainAnswer));
|
|
574
|
-
await this.emit("run.completed", {
|
|
575
|
-
message: plainAnswer,
|
|
576
|
-
plainAnswer: true,
|
|
577
|
-
});
|
|
578
|
-
return {
|
|
579
|
-
sessionId: this.session.sessionId,
|
|
580
|
-
message: plainAnswer,
|
|
581
|
-
};
|
|
1701
|
+
// A finished native refusal/clarification is data for the agent, not a
|
|
1702
|
+
// successful image or an empty-response retry. Keep a bounded budget;
|
|
1703
|
+
// malformed XML still follows strict protocol correction below.
|
|
1704
|
+
if (this.nativeImageReceiver && raw.trim() && !/<\/?(?:agent_response|tool_call|tool_calls|invoke|done|message)(?:\s|>)/i.test(raw)) {
|
|
1705
|
+
protocolErrors += 1;
|
|
1706
|
+
await this.#appendHandoffTranscript(activeHandoff, "assistant", assistantMessage(raw));
|
|
1707
|
+
await this.emit("model.progress", { turn: turnNumber, message: raw });
|
|
1708
|
+
if (protocolErrors >= this.limits.maxProtocolErrors) {
|
|
1709
|
+
throw new ProtocolError("The provider repeatedly returned text without a completed XML answer or an image.");
|
|
1710
|
+
}
|
|
1711
|
+
await this.sendMessage(
|
|
1712
|
+
`<native_image_result type="text">${cdata(raw)}</native_image_result>\n`
|
|
1713
|
+
+ "Your preceding reply contained only this text; no image was received or saved. "
|
|
1714
|
+
+ "Continue the original task based on this result. If generation is unavailable or declined, "
|
|
1715
|
+
+ "explain it or ask the user a specific question in a done=true XML answer. Do not automatically retry the same image request.",
|
|
1716
|
+
{ outboundKind: "native_image_text", runtimeTurn: turnNumber + 1 },
|
|
1717
|
+
);
|
|
1718
|
+
continue;
|
|
582
1719
|
}
|
|
583
1720
|
|
|
1721
|
+
// Absence of an envelope is a protocol failure, not proof of done.
|
|
1722
|
+
// This also covers UI-only text and partial native-artifact responses.
|
|
1723
|
+
await this.adapter.writeDiagnostics?.("protocol-invalid", {
|
|
1724
|
+
turn: turnNumber,
|
|
1725
|
+
assistantMessageId,
|
|
1726
|
+
error: error.message,
|
|
1727
|
+
}).catch(() => null);
|
|
584
1728
|
protocolErrors += 1;
|
|
585
1729
|
await this.emit("protocol.invalid", {
|
|
586
1730
|
message: error.message,
|
|
@@ -591,79 +1735,39 @@ export class AgentRuntime {
|
|
|
591
1735
|
`Protocol failed ${protocolErrors} consecutive times: ${error.message}`,
|
|
592
1736
|
);
|
|
593
1737
|
}
|
|
594
|
-
await this.sendMessage(serializeProtocolError(error)
|
|
1738
|
+
await this.sendMessage(serializeProtocolError(error), {
|
|
1739
|
+
allowAssistantContinuation: true,
|
|
1740
|
+
outboundKind: "protocol_correction",
|
|
1741
|
+
runtimeTurn: turnNumber + 1,
|
|
1742
|
+
});
|
|
595
1743
|
continue;
|
|
596
1744
|
}
|
|
597
1745
|
|
|
598
|
-
// When the run finishes, some models (GLM especially) put their real
|
|
599
|
-
// deliverable AFTER the envelope: a short done/true stub followed by the
|
|
600
|
-
// full markdown/HTML answer. That trailing prose is pure display content
|
|
601
|
-
// (it cannot trigger a tool), so merge it into the final message instead
|
|
602
|
-
// of dropping it. Non-done turns keep the protocol message untouched.
|
|
603
|
-
let finalMessage = parsed.message;
|
|
604
|
-
let usedTrailingProse = false;
|
|
605
|
-
if (parsed.done) {
|
|
606
|
-
const trailing = extractTrailingProse(raw);
|
|
607
|
-
if (trailing) {
|
|
608
|
-
finalMessage = [parsed.message.trim(), trailing]
|
|
609
|
-
.filter(Boolean)
|
|
610
|
-
.join("\n\n");
|
|
611
|
-
usedTrailingProse = true;
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
|
-
|
|
615
1746
|
// Record the assistant's turn in the canonical transcript. The raw XML is
|
|
616
1747
|
// the web rendering; the transcript keeps the plain progress message.
|
|
617
1748
|
if (finalMessage?.trim()) {
|
|
618
|
-
await this
|
|
1749
|
+
await this.#appendHandoffTranscript(
|
|
1750
|
+
activeHandoff,
|
|
1751
|
+
"assistant",
|
|
619
1752
|
assistantMessage(finalMessage),
|
|
620
1753
|
);
|
|
621
1754
|
}
|
|
622
1755
|
|
|
623
1756
|
if (parsed.done) {
|
|
624
|
-
if (!finalMessage.trim()) {
|
|
625
|
-
const error = new ProtocolError(
|
|
626
|
-
"done=true requires a non-empty final message.",
|
|
627
|
-
);
|
|
628
|
-
await this.emit("protocol.invalid", { message: error.message });
|
|
629
|
-
await this.sendMessage(serializeProtocolError(error));
|
|
630
|
-
continue;
|
|
631
|
-
}
|
|
632
|
-
// Some models (DeepSeek especially) write their tool request inside
|
|
633
|
-
// the <message> of a done=true envelope instead of a <tool_call>
|
|
634
|
-
// element. Completing there would swallow the tool call and end the
|
|
635
|
-
// run with raw XML as the "answer". Treat it as a format slip: the
|
|
636
|
-
// protocol-error feedback tells the model where tool calls go, and
|
|
637
|
-
// the normal retry limit still applies.
|
|
638
|
-
if (
|
|
639
|
-
!parsed.toolCall
|
|
640
|
-
&& /<tool_calls[\s>]|<tool_call[\s>]|<invoke[\s>]/i.test(finalMessage)
|
|
641
|
-
) {
|
|
642
|
-
const error = new ProtocolError(
|
|
643
|
-
"Tool calls must use the <tool_call> element, not the <message> text.",
|
|
644
|
-
);
|
|
645
|
-
protocolErrors += 1;
|
|
646
|
-
await this.emit("protocol.invalid", {
|
|
647
|
-
message: error.message,
|
|
648
|
-
count: protocolErrors,
|
|
649
|
-
});
|
|
650
|
-
if (protocolErrors >= this.limits.maxProtocolErrors) {
|
|
651
|
-
throw new ProtocolError(
|
|
652
|
-
`Protocol failed ${protocolErrors} consecutive times: ${error.message}`,
|
|
653
|
-
);
|
|
654
|
-
}
|
|
655
|
-
await this.sendMessage(serializeProtocolError(error));
|
|
656
|
-
continue;
|
|
657
|
-
}
|
|
658
1757
|
// done=true completes the run. A request may be answered directly
|
|
659
1758
|
// (no tool call) or after any number of tools; the runtime does not
|
|
660
1759
|
// second-guess whether "enough" work happened — that is the model's
|
|
661
1760
|
// and the user's call, not a keyword heuristic.
|
|
662
|
-
await this
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
1761
|
+
await this.#queueSessionOperation(() => (
|
|
1762
|
+
this.session.clearPendingAssistantTurn(
|
|
1763
|
+
activeHandoff.handoffId,
|
|
1764
|
+
{
|
|
1765
|
+
phase: "idle",
|
|
1766
|
+
lastMessage: finalMessage,
|
|
1767
|
+
pendingToolResult: null,
|
|
1768
|
+
},
|
|
1769
|
+
)
|
|
1770
|
+
));
|
|
667
1771
|
await this.emit("run.completed", {
|
|
668
1772
|
message: finalMessage,
|
|
669
1773
|
usedTrailingProse,
|
|
@@ -692,6 +1796,11 @@ export class AgentRuntime {
|
|
|
692
1796
|
+ "If you need to take action on the user's machine, request one local tool. "
|
|
693
1797
|
+ "If you need information from the user, ask one specific question in <message> "
|
|
694
1798
|
+ "and set <done>true</done> so control returns to the user.",
|
|
1799
|
+
{
|
|
1800
|
+
allowAssistantContinuation: true,
|
|
1801
|
+
outboundKind: "progress_correction",
|
|
1802
|
+
runtimeTurn: turnNumber + 1,
|
|
1803
|
+
},
|
|
695
1804
|
);
|
|
696
1805
|
continue;
|
|
697
1806
|
}
|
|
@@ -734,16 +1843,24 @@ export class AgentRuntime {
|
|
|
734
1843
|
await this.onEvent?.(completionEvent);
|
|
735
1844
|
// Record the rejected call and its error output so the transcript stays
|
|
736
1845
|
// a faithful, replay-free record of what the model attempted.
|
|
737
|
-
await this
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
await this
|
|
1846
|
+
await this.#appendHandoffTranscript(
|
|
1847
|
+
activeHandoff,
|
|
1848
|
+
"function_call",
|
|
1849
|
+
functionCall({
|
|
1850
|
+
name: normalizedCall.name,
|
|
1851
|
+
args: normalizedCall.args,
|
|
1852
|
+
callId: normalizedCall.id,
|
|
1853
|
+
}),
|
|
1854
|
+
);
|
|
1855
|
+
await this.#appendHandoffTranscript(
|
|
1856
|
+
activeHandoff,
|
|
1857
|
+
"function_output",
|
|
1858
|
+
functionCallOutput({
|
|
1859
|
+
callId: result.callId,
|
|
1860
|
+
output: toolResultOutput(result),
|
|
1861
|
+
}),
|
|
1862
|
+
);
|
|
1863
|
+
await this.sendToolResult(result, { runtimeTurn: turnNumber + 1 });
|
|
747
1864
|
awaitingPendingAcknowledgement = true;
|
|
748
1865
|
await this.emit("tool.result_sent", {
|
|
749
1866
|
id: result.callId,
|
|
@@ -757,16 +1874,21 @@ export class AgentRuntime {
|
|
|
757
1874
|
name: preparedCall.name,
|
|
758
1875
|
args: preparedCall.args,
|
|
759
1876
|
});
|
|
760
|
-
await this
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
1877
|
+
await this.#appendHandoffTranscript(
|
|
1878
|
+
activeHandoff,
|
|
1879
|
+
"function_call",
|
|
1880
|
+
functionCall({
|
|
1881
|
+
name: preparedCall.name,
|
|
1882
|
+
args: preparedCall.args,
|
|
1883
|
+
callId: preparedCall.id,
|
|
1884
|
+
}),
|
|
1885
|
+
);
|
|
765
1886
|
|
|
766
1887
|
const isReadTool = preparedCall.definition.risk === "read";
|
|
767
1888
|
const sideEffect = isReadTool ? null : identity;
|
|
768
1889
|
const fingerprint = identity.fingerprint;
|
|
769
1890
|
let result;
|
|
1891
|
+
let restoredForTool = false;
|
|
770
1892
|
|
|
771
1893
|
if (
|
|
772
1894
|
sideEffect
|
|
@@ -885,32 +2007,46 @@ export class AgentRuntime {
|
|
|
885
2007
|
}
|
|
886
2008
|
|
|
887
2009
|
if (!result) {
|
|
2010
|
+
if (preparedCall.definition.requiresVisibleBrowser) {
|
|
2011
|
+
await this.adapter.restoreWindow?.();
|
|
2012
|
+
restoredForTool = true;
|
|
2013
|
+
}
|
|
888
2014
|
await this.emit("tool.started", {
|
|
889
2015
|
id: preparedCall.id,
|
|
890
2016
|
name: preparedCall.name,
|
|
891
2017
|
});
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
type: "tool.output",
|
|
904
|
-
sessionId: this.session.sessionId,
|
|
905
|
-
timestamp: new Date().toISOString(),
|
|
906
|
-
payload: {
|
|
2018
|
+
try {
|
|
2019
|
+
result = await this.registry.execute(preparedCall, {
|
|
2020
|
+
projectRoot,
|
|
2021
|
+
allowOutside: grants?.allowOutside ?? false,
|
|
2022
|
+
toolTimeoutMs: this.limits.toolTimeoutMs,
|
|
2023
|
+
sessionId: this.session.sessionId,
|
|
2024
|
+
operationKey: identity.operationKey,
|
|
2025
|
+
requestSignature: identity.requestSignature,
|
|
2026
|
+
runAuxiliaryTurn: (options) => this.runAuxiliaryTurn(options),
|
|
2027
|
+
onToolOutput: async (output) => {
|
|
2028
|
+
await this.session.appendToolOutput({
|
|
907
2029
|
id: preparedCall.id,
|
|
908
2030
|
name: preparedCall.name,
|
|
909
2031
|
...output,
|
|
910
|
-
}
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
2032
|
+
});
|
|
2033
|
+
await this.onEvent?.({
|
|
2034
|
+
type: "tool.output",
|
|
2035
|
+
sessionId: this.session.sessionId,
|
|
2036
|
+
timestamp: new Date().toISOString(),
|
|
2037
|
+
payload: {
|
|
2038
|
+
id: preparedCall.id,
|
|
2039
|
+
name: preparedCall.name,
|
|
2040
|
+
...output,
|
|
2041
|
+
},
|
|
2042
|
+
});
|
|
2043
|
+
},
|
|
2044
|
+
});
|
|
2045
|
+
} finally {
|
|
2046
|
+
if (restoredForTool) {
|
|
2047
|
+
await this.adapter.page?.bringToFront?.().catch(() => null);
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
914
2050
|
}
|
|
915
2051
|
|
|
916
2052
|
if (sideEffect) {
|
|
@@ -933,17 +2069,26 @@ export class AgentRuntime {
|
|
|
933
2069
|
}
|
|
934
2070
|
}
|
|
935
2071
|
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
2072
|
+
this.#assertProviderReadyForMessage();
|
|
2073
|
+
|
|
2074
|
+
await this.#appendHandoffTranscript(
|
|
2075
|
+
activeHandoff,
|
|
2076
|
+
"function_output",
|
|
2077
|
+
functionCallOutput({
|
|
2078
|
+
callId: result.callId,
|
|
2079
|
+
output: toolResultOutput(result),
|
|
2080
|
+
}),
|
|
2081
|
+
);
|
|
2082
|
+
await this.sendToolResult(result, { runtimeTurn: turnNumber + 1 });
|
|
941
2083
|
awaitingPendingAcknowledgement = true;
|
|
942
2084
|
await this.emit("tool.result_sent", {
|
|
943
2085
|
id: result.callId,
|
|
944
2086
|
name: result.name,
|
|
945
2087
|
ok: result.ok,
|
|
946
2088
|
});
|
|
2089
|
+
if (restoredForTool) {
|
|
2090
|
+
await this.adapter.minimizeWindow?.();
|
|
2091
|
+
}
|
|
947
2092
|
}
|
|
948
2093
|
|
|
949
2094
|
}
|