sealbridge 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +22 -0
- package/README.md +104 -3
- package/dist/cli/dist/approval-summary.d.ts +2 -0
- package/dist/cli/dist/approval-summary.js +32 -0
- package/dist/cli/dist/approval-summary.js.map +1 -0
- package/dist/cli/dist/error-messages.d.ts +2 -0
- package/dist/cli/dist/error-messages.js +24 -0
- package/dist/cli/dist/error-messages.js.map +1 -0
- package/dist/cli/dist/index.js +2 -1
- package/dist/cli/dist/index.js.map +1 -1
- package/dist/cli/dist/invite-share.js +2 -8
- package/dist/cli/dist/invite-share.js.map +1 -1
- package/dist/cli/dist/network.js +87 -11
- package/dist/cli/dist/network.js.map +1 -1
- package/dist/cli/dist/task-input.d.ts +2 -0
- package/dist/cli/dist/task-input.js +19 -0
- package/dist/cli/dist/task-input.js.map +1 -0
- package/dist/cli/dist/wake-hook.d.ts +3 -0
- package/dist/cli/dist/wake-hook.js +32 -0
- package/dist/cli/dist/wake-hook.js.map +1 -0
- package/dist/package.json +1 -1
- package/dist/packages/adapters/dist/command.d.ts +2 -0
- package/dist/packages/adapters/dist/command.js +13 -0
- package/dist/packages/adapters/dist/command.js.map +1 -1
- package/dist/packages/core/dist/capability.js +7 -2
- package/dist/packages/core/dist/capability.js.map +1 -1
- package/dist/packages/core/dist/crypto.js +8 -2
- package/dist/packages/core/dist/crypto.js.map +1 -1
- package/dist/packages/core/dist/discovery.d.ts +5 -0
- package/dist/packages/core/dist/discovery.js +29 -13
- package/dist/packages/core/dist/discovery.js.map +1 -1
- package/dist/packages/core/dist/revocation.d.ts +4 -1
- package/dist/packages/core/dist/revocation.js +29 -5
- package/dist/packages/core/dist/revocation.js.map +1 -1
- package/dist/packages/relay/dist/fetch.d.ts +28 -1
- package/dist/packages/relay/dist/fetch.js +195 -4
- package/dist/packages/relay/dist/fetch.js.map +1 -1
- package/dist/packages/relay/dist/index.d.ts +16 -5
- package/dist/packages/relay/dist/index.js +243 -95
- package/dist/packages/relay/dist/index.js.map +1 -1
- package/dist/packages/relay/dist/journal.d.ts +7 -0
- package/dist/packages/relay/dist/journal.js +33 -4
- package/dist/packages/relay/dist/journal.js.map +1 -1
- package/dist/packages/relay/dist/notifications.d.ts +63 -0
- package/dist/packages/relay/dist/notifications.js +55 -0
- package/dist/packages/relay/dist/notifications.js.map +1 -0
- package/dist/packages/relay/dist/retry.d.ts +4 -1
- package/dist/packages/relay/dist/retry.js +27 -7
- package/dist/packages/relay/dist/retry.js.map +1 -1
- package/dist/packages/relay/dist/worker.d.ts +7 -0
- package/dist/packages/relay/dist/worker.js +57 -8
- package/dist/packages/relay/dist/worker.js.map +1 -1
- package/dist/packages/runtime/dist/a2a.js +1 -1
- package/dist/packages/runtime/dist/approval-providers.d.ts +2 -0
- package/dist/packages/runtime/dist/approval-providers.js +2 -2
- package/dist/packages/runtime/dist/approval-providers.js.map +1 -1
- package/dist/packages/runtime/dist/mcp-server.js +17 -1
- package/dist/packages/runtime/dist/mcp-server.js.map +1 -1
- package/dist/packages/runtime/dist/mcp-tool-catalog.js +16 -0
- package/dist/packages/runtime/dist/mcp-tool-catalog.js.map +1 -1
- package/dist/packages/runtime/dist/session-audit.d.ts +22 -3
- package/dist/packages/runtime/dist/session-audit.js +128 -23
- package/dist/packages/runtime/dist/session-audit.js.map +1 -1
- package/dist/packages/runtime/dist/session-timeouts.d.ts +33 -0
- package/dist/packages/runtime/dist/session-timeouts.js +62 -0
- package/dist/packages/runtime/dist/session-timeouts.js.map +1 -0
- package/dist/packages/runtime/dist/session.d.ts +52 -17
- package/dist/packages/runtime/dist/session.js +466 -94
- package/dist/packages/runtime/dist/session.js.map +1 -1
- package/dist/packages/runtime/dist/task-journal.d.ts +36 -0
- package/dist/packages/runtime/dist/task-journal.js +104 -0
- package/dist/packages/runtime/dist/task-journal.js.map +1 -0
- package/package.json +1 -1
|
@@ -5,14 +5,16 @@ import { existsSync, openSync, closeSync, fstatSync, fchmodSync, appendFileSync,
|
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { tmpdir } from 'node:os';
|
|
7
7
|
import { randomUUID } from 'node:crypto';
|
|
8
|
-
import { createContract, createCapability, createSignedAgentCard, createInviteToken, parseAndVerifyInviteToken, validateRelayEndpointUrl, signPayload, verifySigned, verifyEnvelope, verifyContract, verifyApproval, validateManifest, validateOutput, verifyCapabilityChain, capabilityIssuerControls, findDiscoveredAgentCard, discoverAgentCards, schemaNarrows, digest, openEnvelope,
|
|
9
|
-
import { verifyAudit } from '../../ledger/dist/index.js';
|
|
10
|
-
import { HttpRelayClient } from '../../relay/dist/index.js';
|
|
8
|
+
import { createContract, createCapability, createSignedAgentCard, createInviteToken, parseAndVerifyInviteToken, isInviteTarget, validateRelayEndpointUrl, signPayload, verifySigned, verifyEnvelope, verifyContract, verifyApproval, validateManifest, validateOutput, verifyCapabilityChain, capabilityIssuerControls, findDiscoveredAgentCard, discoverAgentCards, schemaNarrows, digest, openEnvelope, resolveDid, createRevocationNotice, verifyRevocationNotice, RevocationRegistry, signEnvelope, DEFAULT_CLOCK_SKEW_MS, OPEN_POLICY_FIELDS, } from '../../core/dist/index.js';
|
|
9
|
+
import { parseJsonl, verifyAudit } from '../../ledger/dist/index.js';
|
|
10
|
+
import { HttpRelayClient, waitForRelayRetry, relayDiagnostic, relayDeadline } from '../../relay/dist/index.js';
|
|
11
11
|
import { InteractiveCliApprovalProvider, ProposalRejectedError, evaluateContractAcceptance, } from './approval-providers.js';
|
|
12
12
|
import { OwnerGateway } from './gateway.js';
|
|
13
13
|
import { MCPProxy } from './mcp.js';
|
|
14
14
|
import { VerificationEngine, createDelivery } from './verifier.js';
|
|
15
|
-
import { asJson, receiveRelay, acknowledgeRelay, SessionWire } from './session-audit.js';
|
|
15
|
+
import { asJson, receiveRelay, acknowledgeRelay, PeerRejectedError, SessionWire } from './session-audit.js';
|
|
16
|
+
import { SessionTimeouts, validateSessionTimeouts, resolveSessionTimeouts } from './session-timeouts.js';
|
|
17
|
+
import { TaskJournal } from './task-journal.js';
|
|
16
18
|
const policy = { requiredGates: [], autoApprove: [] };
|
|
17
19
|
/** Daemon workers stay finite unless the owner raises the cap. */
|
|
18
20
|
export const DEFAULT_MAX_CONCURRENT_TASKS = 8;
|
|
@@ -58,15 +60,31 @@ function transport(relay, owner, security) {
|
|
|
58
60
|
? new HttpRelayClient(relay.replace(/\/$/, ''), owner, security)
|
|
59
61
|
: relay;
|
|
60
62
|
client.setRecipientKey?.(owner);
|
|
61
|
-
|
|
63
|
+
const bound = client.withRecipient?.(owner) ?? client;
|
|
64
|
+
return {
|
|
65
|
+
get connectionStatus() { return bound.connectionStatus; },
|
|
66
|
+
send: (envelope) => relayDeadline(Promise.resolve(bound.send(envelope)), security?.requestTimeoutMs ?? 45_000),
|
|
67
|
+
receive: async (did, waitMs, signal, filter) => {
|
|
68
|
+
const controller = new AbortController();
|
|
69
|
+
const readSignal = AbortSignal.any([controller.signal, ...(signal ? [signal] : [])]);
|
|
70
|
+
try {
|
|
71
|
+
return await relayDeadline(Promise.resolve(bound.receive(did, waitMs, readSignal, filter)), security?.requestTimeoutMs ?? 45_000, readSignal);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
controller.abort(error);
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
...(bound.acknowledgeDelivery ? { acknowledgeDelivery: (did, id) => relayDeadline(Promise.resolve(bound.acknowledgeDelivery(did, id)), security?.requestTimeoutMs ?? 45_000) } : {}),
|
|
79
|
+
...(bound.deliveryStatus ? { deliveryStatus: (envelope, signal) => relayDeadline(Promise.resolve(bound.deliveryStatus(envelope, signal)), security?.requestTimeoutMs ?? 45_000, signal) } : {}),
|
|
80
|
+
...(bound.register ? { register: (card) => bound.register(card) } : {}),
|
|
81
|
+
...(bound.discover ? { discover: () => relayDeadline(Promise.resolve(bound.discover()), security?.requestTimeoutMs ?? 45_000) } : {}),
|
|
82
|
+
};
|
|
62
83
|
}
|
|
63
84
|
function freshAudit(path) {
|
|
64
85
|
if (path && existsSync(path))
|
|
65
86
|
throw new Error('AUDIT_FILE_ALREADY_EXISTS');
|
|
66
87
|
}
|
|
67
|
-
function sessionSignal(timeout, signal) {
|
|
68
|
-
return AbortSignal.any([AbortSignal.timeout(timeout), ...(signal ? [signal] : [])]);
|
|
69
|
-
}
|
|
70
88
|
/** Cleartext is rejected unless the owner explicitly opts out for a legacy fixture. */
|
|
71
89
|
function sessionDecryptOptions(allowUnencryptedEnvelopes) {
|
|
72
90
|
return { requireEncrypted: allowUnencryptedEnvelopes !== true };
|
|
@@ -74,6 +92,12 @@ function sessionDecryptOptions(allowUnencryptedEnvelopes) {
|
|
|
74
92
|
/** Per-task queue so one session cannot dequeue another task's envelopes. */
|
|
75
93
|
class TaskMailbox {
|
|
76
94
|
queue = [];
|
|
95
|
+
error;
|
|
96
|
+
close(error) {
|
|
97
|
+
this.error ??= error;
|
|
98
|
+
for (const wake of this.waiters.splice(0))
|
|
99
|
+
wake();
|
|
100
|
+
}
|
|
77
101
|
waiters = [];
|
|
78
102
|
push(envelope) {
|
|
79
103
|
this.queue.push(envelope);
|
|
@@ -81,6 +105,8 @@ class TaskMailbox {
|
|
|
81
105
|
}
|
|
82
106
|
async take(signal) {
|
|
83
107
|
for (;;) {
|
|
108
|
+
if (this.error)
|
|
109
|
+
throw this.error;
|
|
84
110
|
signal?.throwIfAborted();
|
|
85
111
|
const next = this.queue.shift();
|
|
86
112
|
if (next)
|
|
@@ -95,7 +121,7 @@ class TaskMailbox {
|
|
|
95
121
|
this.waiters = this.waiters.filter((waiter) => waiter !== onWake);
|
|
96
122
|
error ? reject(error) : resolve();
|
|
97
123
|
};
|
|
98
|
-
const onAbort = () => finish(new Error('SESSION_ABORTED'));
|
|
124
|
+
const onAbort = () => finish(signal?.reason instanceof Error ? signal.reason : new Error('SESSION_ABORTED'));
|
|
99
125
|
const onWake = () => finish();
|
|
100
126
|
this.waiters.push(onWake);
|
|
101
127
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
@@ -110,6 +136,7 @@ class TaskMailbox {
|
|
|
110
136
|
function mailboxRelay(relay, mailbox) {
|
|
111
137
|
return {
|
|
112
138
|
send: (envelope) => relay.send(envelope),
|
|
139
|
+
...(relay.deliveryStatus ? { deliveryStatus: (envelope, signal) => relay.deliveryStatus(envelope, signal) } : {}),
|
|
113
140
|
...(relay.acknowledgeDelivery ? { acknowledgeDelivery: (did, id) => relay.acknowledgeDelivery(did, id) } : {}),
|
|
114
141
|
receive: (_recipientDid, _waitMs, signal) => mailbox.take(signal),
|
|
115
142
|
};
|
|
@@ -189,6 +216,7 @@ class RecipientInbox {
|
|
|
189
216
|
wake;
|
|
190
217
|
readAbort;
|
|
191
218
|
started = false;
|
|
219
|
+
error;
|
|
192
220
|
constructor(relay, did) {
|
|
193
221
|
this.relay = relay;
|
|
194
222
|
this.did = did;
|
|
@@ -204,6 +232,8 @@ class RecipientInbox {
|
|
|
204
232
|
return true;
|
|
205
233
|
}
|
|
206
234
|
attach(taskId) {
|
|
235
|
+
if (this.error)
|
|
236
|
+
throw this.error;
|
|
207
237
|
const existing = this.mailboxes.get(taskId);
|
|
208
238
|
if (existing)
|
|
209
239
|
return existing;
|
|
@@ -258,8 +288,15 @@ class RecipientInbox {
|
|
|
258
288
|
}
|
|
259
289
|
this.mailboxes.get(envelope.payload.taskId)?.push(envelope);
|
|
260
290
|
}
|
|
261
|
-
catch {
|
|
262
|
-
|
|
291
|
+
catch (error) {
|
|
292
|
+
if (controller.signal.aborted)
|
|
293
|
+
continue;
|
|
294
|
+
this.error = relayDiagnostic(error);
|
|
295
|
+
controller.abort(this.error);
|
|
296
|
+
for (const mailbox of this.mailboxes.values())
|
|
297
|
+
mailbox.close(this.error);
|
|
298
|
+
unregisterRequesterInbox(this.did, this);
|
|
299
|
+
return;
|
|
263
300
|
}
|
|
264
301
|
}
|
|
265
302
|
}
|
|
@@ -324,6 +361,8 @@ export class WorkerNode {
|
|
|
324
361
|
acceptance;
|
|
325
362
|
completed = 0;
|
|
326
363
|
activeTasks = 0;
|
|
364
|
+
taskJournal;
|
|
365
|
+
reconciled = false;
|
|
327
366
|
proposalTimes = new Map();
|
|
328
367
|
mailboxes = new Map();
|
|
329
368
|
forwardedQueue = [];
|
|
@@ -340,6 +379,13 @@ export class WorkerNode {
|
|
|
340
379
|
revocation;
|
|
341
380
|
constructor(options) {
|
|
342
381
|
this.options = options;
|
|
382
|
+
validateSessionTimeouts(options);
|
|
383
|
+
if (options.inviteTtlSeconds !== undefined && (!Number.isSafeInteger(options.inviteTtlSeconds) ||
|
|
384
|
+
options.inviteTtlSeconds < 1 || options.inviteTtlSeconds > 2592000))
|
|
385
|
+
throw new Error('INVALID_INVITE_TTL');
|
|
386
|
+
const persistencePath = options.inboxPath ?? options.auditPath;
|
|
387
|
+
if (persistencePath)
|
|
388
|
+
this.taskJournal = new TaskJournal(persistencePath + '.tasks.jsonl', options.owner);
|
|
343
389
|
if (options.dailyLimit && (!options.dailyLimit.path || !Number.isSafeInteger(options.dailyLimit.maxTasks) || options.dailyLimit.maxTasks < 1))
|
|
344
390
|
throw new Error('INVALID_DAILY_LIMIT');
|
|
345
391
|
if (options.accessMode !== undefined && !['invite-only', 'open'].includes(options.accessMode))
|
|
@@ -354,6 +400,7 @@ export class WorkerNode {
|
|
|
354
400
|
this.relay = this.revocationAwareRelay(transport(options.relay, options.owner, {
|
|
355
401
|
allowPrivateRelay: options.allowPrivateRelay,
|
|
356
402
|
lookup: options.lookup,
|
|
403
|
+
requestTimeoutMs: options.connectTimeoutMs,
|
|
357
404
|
}));
|
|
358
405
|
this.approvalProvider =
|
|
359
406
|
options.approvalProvider ?? new InteractiveCliApprovalProvider(options.owner);
|
|
@@ -365,11 +412,14 @@ export class WorkerNode {
|
|
|
365
412
|
? options.relay
|
|
366
413
|
: (options.relayUrl ?? 'http://in-memory.invalid'),
|
|
367
414
|
skills: options.adapter.skills,
|
|
415
|
+
...(options.inviteTtlSeconds !== undefined ? {
|
|
416
|
+
expiresAt: new Date(Date.now() + options.inviteTtlSeconds * 1000).toISOString(),
|
|
417
|
+
} : {}),
|
|
368
418
|
...(options.accessMode ? { accessMode: options.accessMode } : {}),
|
|
369
419
|
...(options.openPolicy !== undefined ? { openPolicy: options.openPolicy } : {}),
|
|
370
420
|
}, options.owner, agentDid === options.owner.did ? undefined : options.agentKey);
|
|
371
421
|
}
|
|
372
|
-
invite(expiresInSeconds =
|
|
422
|
+
invite(expiresInSeconds = this.options.inviteTtlSeconds ?? 86400) {
|
|
373
423
|
return createInviteToken({ agentCard: this.agentCard, expiresInSeconds }, this.options.owner);
|
|
374
424
|
}
|
|
375
425
|
async register() {
|
|
@@ -377,6 +427,7 @@ export class WorkerNode {
|
|
|
377
427
|
}
|
|
378
428
|
stop() {
|
|
379
429
|
this.controller.abort('WORKER_STOPPED');
|
|
430
|
+
this.listenerStatus('stopped');
|
|
380
431
|
}
|
|
381
432
|
/** Public registration is opt-in, including for PolicyAutoApprovalProvider. */
|
|
382
433
|
publishes() {
|
|
@@ -431,9 +482,23 @@ export class WorkerNode {
|
|
|
431
482
|
return undefined;
|
|
432
483
|
}
|
|
433
484
|
}
|
|
485
|
+
status;
|
|
486
|
+
listenerStatus(status, error) {
|
|
487
|
+
if (this.status === status && !error)
|
|
488
|
+
return;
|
|
489
|
+
this.status = status;
|
|
490
|
+
this.options.onStatus?.(status, error);
|
|
491
|
+
}
|
|
492
|
+
listen(signal) {
|
|
493
|
+
return receiveRelay(this.relay, this.options.owner.did, signal, { role: 'worker' }, (status) => this.listenerStatus(status === 'connected' ? this.relay.connectionStatus ?? status : status))
|
|
494
|
+
.catch((error) => { if (signal.aborted)
|
|
495
|
+
throw error; throw relayDiagnostic(error); });
|
|
496
|
+
}
|
|
434
497
|
async start(options = {}) {
|
|
498
|
+
// Register first so immediate discovery can resolve owner and agent aliases.
|
|
435
499
|
if (this.publishes())
|
|
436
500
|
await this.register();
|
|
501
|
+
await this.reconcileTasks();
|
|
437
502
|
const idleSignal = AbortSignal.any([
|
|
438
503
|
this.controller.signal,
|
|
439
504
|
...(options.signal ? [options.signal] : []),
|
|
@@ -447,11 +512,12 @@ export class WorkerNode {
|
|
|
447
512
|
try {
|
|
448
513
|
while (!idleSignal.aborted) {
|
|
449
514
|
// revocationAwareRelay ingests a leading REVOKE before this read returns.
|
|
450
|
-
const envelope = await
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
515
|
+
const envelope = await this.listen(idleSignal);
|
|
516
|
+
if (await this.recoverProposal(envelope)) {
|
|
517
|
+
if (options.once)
|
|
518
|
+
return this.lastResult;
|
|
454
519
|
continue;
|
|
520
|
+
}
|
|
455
521
|
if (this.deliver(envelope))
|
|
456
522
|
continue;
|
|
457
523
|
const job = this.beginTask(envelope, idleSignal, options.once === true);
|
|
@@ -466,35 +532,140 @@ export class WorkerNode {
|
|
|
466
532
|
}
|
|
467
533
|
}
|
|
468
534
|
catch (error) {
|
|
469
|
-
if (!idleSignal.aborted)
|
|
535
|
+
if (!idleSignal.aborted) {
|
|
536
|
+
// A rejected one-shot task may be followed by another start on this node.
|
|
537
|
+
// Only a terminal listener failure shuts down the worker and all of its tasks.
|
|
538
|
+
if (error instanceof Error && 'code' in error && error.code === 'RELAY_TERMINAL') {
|
|
539
|
+
this.listenerStatus('unavailable', error);
|
|
540
|
+
for (const mailbox of this.mailboxes.values())
|
|
541
|
+
mailbox.close(error);
|
|
542
|
+
this.controller.abort(error);
|
|
543
|
+
}
|
|
470
544
|
throw error;
|
|
545
|
+
}
|
|
471
546
|
}
|
|
472
547
|
finally {
|
|
548
|
+
if (this.status !== 'unavailable')
|
|
549
|
+
this.listenerStatus('stopped');
|
|
473
550
|
unregister();
|
|
474
551
|
await Promise.allSettled([...jobs]);
|
|
475
552
|
}
|
|
476
553
|
return this.lastResult;
|
|
477
554
|
}
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
555
|
+
async notifyOutcome(record, outcome = record.outcome) {
|
|
556
|
+
const send = (recovery) => this.relay.send(signEnvelope({ taskId: record.taskId,
|
|
557
|
+
recipientDid: record.requesterDid, seq: 0, eventType: 'ERROR',
|
|
558
|
+
payload: asJson({ code: outcome.code, recovery }),
|
|
559
|
+
}, this.options.owner, { encrypt: true }));
|
|
560
|
+
const bytes = outcome.result ? Buffer.from(JSON.stringify(outcome.result)) : undefined;
|
|
561
|
+
// Cached audits can exceed a single relay envelope even when every original event fitted.
|
|
562
|
+
if (bytes && bytes.length > 48 * 1024) {
|
|
563
|
+
const size = 24 * 1024, count = Math.ceil(bytes.length / size), resultDigest = digest(outcome.result);
|
|
564
|
+
for (let index = 0; index < count; index++) {
|
|
565
|
+
await send({ proposalDigest: record.proposalDigest, part: { index, count, resultDigest,
|
|
566
|
+
data: bytes.subarray(index * size, (index + 1) * size).toString('base64') } });
|
|
567
|
+
// Yield to inbox readers so a cached audit does not fill the bounded relay queue.
|
|
568
|
+
if (index + 1 < count)
|
|
569
|
+
await waitForRelayRetry(10, this.controller.signal);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
else {
|
|
573
|
+
await send({ proposalDigest: record.proposalDigest, ...(outcome.result ? { result: outcome.result } : {}) });
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
async reconcileTasks() {
|
|
577
|
+
if (this.reconciled || !this.taskJournal)
|
|
578
|
+
return;
|
|
579
|
+
this.taskJournal.load();
|
|
580
|
+
for (const saved of this.taskJournal.tasks.values()) {
|
|
581
|
+
let record = saved;
|
|
582
|
+
if (record.phase !== 'terminal') {
|
|
583
|
+
const result = this.sealedTaskResult(record);
|
|
584
|
+
// Adapters expose executeTask only. Unknown execution/tool results cannot be retried safely.
|
|
585
|
+
record = { ...record, phase: 'terminal', outcome: result
|
|
586
|
+
? { code: result.state === 'SETTLED' ? 'TASK_SETTLED' : 'TASK_CANCELLED', result }
|
|
587
|
+
: { code: 'TASK_INTERRUPTED' }, notificationPending: true };
|
|
588
|
+
this.taskJournal.save(record);
|
|
589
|
+
if (!result)
|
|
590
|
+
this.options.onEvent?.('INTERRUPTED', record.taskId, { code: 'TASK_INTERRUPTED' });
|
|
591
|
+
}
|
|
592
|
+
if (record.notificationPending) {
|
|
593
|
+
await this.notifyOutcome(record);
|
|
594
|
+
this.taskJournal.save({ ...record, notificationPending: false });
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
this.reconciled = true;
|
|
598
|
+
}
|
|
599
|
+
sealedTaskResult(record) {
|
|
600
|
+
if (!record.auditPath)
|
|
601
|
+
return;
|
|
602
|
+
try {
|
|
603
|
+
const bundle = parseJsonl(readFileSync(record.auditPath, 'utf8'));
|
|
604
|
+
if (!verifyAudit(bundle, { taskId: record.taskId, actors: [record.requesterDid, this.options.owner.did] }).valid)
|
|
605
|
+
return;
|
|
606
|
+
const proposal = bundle.entries.find((entry) => entry.body.eventType === 'PROPOSE')?.body.payload;
|
|
607
|
+
if (!proposal || digest({ manifest: proposal.proposal.payload, input: proposal.input }) !== record.proposalDigest)
|
|
608
|
+
return;
|
|
609
|
+
const last = bundle.entries.at(-1)?.body.eventType;
|
|
610
|
+
if (last === 'CANCEL')
|
|
611
|
+
return { taskId: record.taskId, state: 'CANCELLED', deliverable: null, bundle };
|
|
612
|
+
const delivery = bundle.entries.find((entry) => entry.body.eventType === 'DELIVER')?.body.payload;
|
|
613
|
+
if (last === 'SETTLED' && delivery)
|
|
614
|
+
return { taskId: record.taskId, state: 'SETTLED', deliverable: delivery.artifact.payload.data, bundle };
|
|
615
|
+
}
|
|
616
|
+
catch { /* An unsealed or partial audit is evidence to preserve, not authorization to rerun. */ }
|
|
617
|
+
return undefined;
|
|
618
|
+
}
|
|
619
|
+
async recoverProposal(envelope) {
|
|
620
|
+
if (!this.taskJournal || envelope.payload.eventType !== 'PROPOSE' || envelope.payload.seq !== 0 ||
|
|
481
621
|
!verifyEnvelope(envelope, this.options.owner.did))
|
|
482
622
|
return false;
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
623
|
+
let data;
|
|
624
|
+
try {
|
|
625
|
+
data = openEnvelope(envelope, this.options.owner, sessionDecryptOptions(this.options.allowUnencryptedEnvelopes));
|
|
626
|
+
}
|
|
627
|
+
catch {
|
|
628
|
+
return false;
|
|
629
|
+
}
|
|
630
|
+
const manifest = data?.proposal?.payload;
|
|
631
|
+
if (!manifest || !verifySigned(data.proposal, envelope.payload.senderDid) ||
|
|
632
|
+
manifest.taskId !== envelope.payload.taskId || manifest.requester?.ownerDid !== envelope.payload.senderDid)
|
|
633
|
+
return false;
|
|
634
|
+
const proposalDigest = digest({ manifest, input: data.input });
|
|
635
|
+
let record = this.taskJournal.tasks.get(manifest.taskId);
|
|
636
|
+
// Older inboxes have intake evidence but no execution checkpoint. Never execute those tasks again.
|
|
637
|
+
if (!record && this.options.inboxPath && existsSync(this.options.inboxPath)) {
|
|
638
|
+
for (const line of readFileSync(this.options.inboxPath, 'utf8').split('\n')) {
|
|
639
|
+
let old;
|
|
640
|
+
try {
|
|
641
|
+
old = JSON.parse(line);
|
|
642
|
+
}
|
|
643
|
+
catch {
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
if (old?.taskId === manifest.taskId && old.requesterOwnerDid === envelope.payload.senderDid) {
|
|
647
|
+
record = { version: 1, taskId: manifest.taskId, requesterDid: envelope.payload.senderDid,
|
|
648
|
+
envelopeId: envelope.payload.envelopeId, proposalDigest, phase: 'terminal', invocations: [],
|
|
649
|
+
outcome: { code: 'TASK_INTERRUPTED' } };
|
|
650
|
+
this.taskJournal.save(record);
|
|
651
|
+
break;
|
|
493
652
|
}
|
|
494
653
|
}
|
|
495
654
|
}
|
|
496
|
-
if (!
|
|
655
|
+
if (!record)
|
|
497
656
|
return false;
|
|
657
|
+
this.lastResult = undefined;
|
|
658
|
+
if (record.requesterDid !== envelope.payload.senderDid || record.proposalDigest !== proposalDigest) {
|
|
659
|
+
await this.notifyOutcome({ ...record, requesterDid: envelope.payload.senderDid, proposalDigest }, { code: 'TASK_ID_CONFLICT' });
|
|
660
|
+
}
|
|
661
|
+
else if (record.outcome) {
|
|
662
|
+
await this.notifyOutcome(record);
|
|
663
|
+
this.lastResult = record.outcome.result;
|
|
664
|
+
}
|
|
665
|
+
else if (!this.mailboxes.has(record.taskId)) {
|
|
666
|
+
throw new Error('TASK_RECOVERY_REQUIRED');
|
|
667
|
+
}
|
|
668
|
+
// The outcome must be sent successfully before a retried intake is removed.
|
|
498
669
|
await acknowledgeRelay(this.relay, envelope);
|
|
499
670
|
return true;
|
|
500
671
|
}
|
|
@@ -571,7 +742,7 @@ export class WorkerNode {
|
|
|
571
742
|
});
|
|
572
743
|
try {
|
|
573
744
|
while (!idleSignal.aborted && failure === undefined) {
|
|
574
|
-
const incoming =
|
|
745
|
+
const incoming = this.listen(AbortSignal.any([idleSignal, wait.signal])).then((envelope) => envelope, (error) => {
|
|
575
746
|
if (!idleSignal.aborted && !wait.signal.aborted)
|
|
576
747
|
failure = error;
|
|
577
748
|
return undefined;
|
|
@@ -583,6 +754,8 @@ export class WorkerNode {
|
|
|
583
754
|
// not delivered here. The next session envelope is what remains.
|
|
584
755
|
if (!envelope)
|
|
585
756
|
return;
|
|
757
|
+
if (await this.recoverProposal(envelope))
|
|
758
|
+
continue;
|
|
586
759
|
if (!this.deliver(envelope) && verifyEnvelope(envelope, this.options.owner.did))
|
|
587
760
|
this.options.onEvent?.('REJECTED', envelope.payload.taskId, { code: 'INVALID_SESSION_ENVELOPE' });
|
|
588
761
|
}
|
|
@@ -594,14 +767,17 @@ export class WorkerNode {
|
|
|
594
767
|
throw failure;
|
|
595
768
|
}
|
|
596
769
|
async runOnce(signal) {
|
|
770
|
+
await this.reconcileTasks();
|
|
597
771
|
const idleSignal = AbortSignal.any([this.controller.signal, ...(signal ? [signal] : [])]);
|
|
598
772
|
// One or more leading REVOKE envelopes are consumed before the task PROPOSE.
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
773
|
+
const first = await receiveRelay(this.relay, this.options.owner.did, idleSignal, {
|
|
774
|
+
role: 'worker',
|
|
775
|
+
});
|
|
776
|
+
if (await this.recoverProposal(first)) {
|
|
777
|
+
if (this.lastResult?.taskId === first.payload.taskId)
|
|
778
|
+
return this.lastResult;
|
|
779
|
+
throw new Error(this.taskJournal?.tasks.get(first.payload.taskId)?.outcome?.code ?? 'TASK_INTERRUPTED');
|
|
780
|
+
}
|
|
605
781
|
return this.executeTask(first, idleSignal, this.relay);
|
|
606
782
|
}
|
|
607
783
|
async executeTask(first, idleSignal, relay, earlyRejection) {
|
|
@@ -614,6 +790,16 @@ export class WorkerNode {
|
|
|
614
790
|
let data;
|
|
615
791
|
let m;
|
|
616
792
|
let auditPath;
|
|
793
|
+
let checkpoint;
|
|
794
|
+
const saveCheckpoint = (patch) => {
|
|
795
|
+
if (!checkpoint || !this.taskJournal)
|
|
796
|
+
return;
|
|
797
|
+
checkpoint = { ...checkpoint, ...patch };
|
|
798
|
+
this.taskJournal.save(checkpoint);
|
|
799
|
+
};
|
|
800
|
+
const saveOutcome = (outcome) => {
|
|
801
|
+
saveCheckpoint({ phase: 'terminal', outcome, notificationPending: true });
|
|
802
|
+
};
|
|
617
803
|
try {
|
|
618
804
|
data = openEnvelope(first, this.options.owner, sessionDecryptOptions(this.options.allowUnencryptedEnvelopes));
|
|
619
805
|
const manifest = data?.proposal?.payload;
|
|
@@ -634,6 +820,12 @@ export class WorkerNode {
|
|
|
634
820
|
: this.options.auditPath.replace(/(?:\.jsonl)?$/, '.' + m.taskId + '.jsonl')
|
|
635
821
|
: undefined;
|
|
636
822
|
freshAudit(auditPath);
|
|
823
|
+
if (this.taskJournal) {
|
|
824
|
+
checkpoint = { version: 1, taskId: m.taskId, requesterDid: m.requester.ownerDid,
|
|
825
|
+
envelopeId: first.payload.envelopeId, proposalDigest: digest({ manifest: m, input: data.input }),
|
|
826
|
+
phase: 'received', auditPath, invocations: [] };
|
|
827
|
+
this.taskJournal.save(checkpoint);
|
|
828
|
+
}
|
|
637
829
|
}
|
|
638
830
|
catch (e) {
|
|
639
831
|
this.reportRejection(first.payload.taskId, e);
|
|
@@ -651,11 +843,25 @@ export class WorkerNode {
|
|
|
651
843
|
}
|
|
652
844
|
throw e;
|
|
653
845
|
}
|
|
654
|
-
const
|
|
846
|
+
const timeouts = new SessionTimeouts(this.options, m.deadline, idleSignal);
|
|
847
|
+
const activeSignal = timeouts.signal;
|
|
655
848
|
const wire = new SessionWire(relay, this.options.owner, m.requester.ownerDid, m.taskId, [m.requester.ownerDid, this.options.owner.did], activeSignal, auditPath, this.options.allowUnencryptedEnvelopes === true);
|
|
656
849
|
const cancelController = new AbortController();
|
|
657
850
|
wire.earlyCancel = () => cancelController.abort('CANCELLED');
|
|
851
|
+
let heartbeat;
|
|
852
|
+
let heartbeatStopped = false;
|
|
853
|
+
const sendHeartbeat = async () => {
|
|
854
|
+
try {
|
|
855
|
+
await wire.heartbeat();
|
|
856
|
+
}
|
|
857
|
+
catch { /* The supervised reader reports relay availability. */ }
|
|
858
|
+
if (!heartbeatStopped && !activeSignal.aborted) {
|
|
859
|
+
heartbeat = setTimeout(() => void sendHeartbeat(), this.options.heartbeatIntervalMs ?? 15_000);
|
|
860
|
+
heartbeat.unref();
|
|
861
|
+
}
|
|
862
|
+
};
|
|
658
863
|
let gateway, workspace;
|
|
864
|
+
let preserveWorkspace = false;
|
|
659
865
|
let trackedTaskId;
|
|
660
866
|
// Progress metrics are applied synchronously. The tail only serializes wire frames.
|
|
661
867
|
let progressTail = Promise.resolve();
|
|
@@ -696,20 +902,30 @@ export class WorkerNode {
|
|
|
696
902
|
validateManifest(m);
|
|
697
903
|
if (!validateOutput(m.io.inputSchema, data.input))
|
|
698
904
|
throw new Error('INVALID_PROPOSAL');
|
|
905
|
+
if (this.options.executionTimeoutMs !== undefined && m.budget.maxWallClockSeconds * 1000 > this.options.executionTimeoutMs)
|
|
906
|
+
throw diagnosticError('INCOMPATIBLE_TIMEOUTS', 'budget.maxWallClockSeconds', this.options.executionTimeoutMs / 1000, m.budget.maxWallClockSeconds, 'Reduce the execution budget or raise the worker execution timeout before accepting.');
|
|
907
|
+
if (this.options.taskTimeoutMs !== undefined && m.budget.maxWallClockSeconds * 1000 > this.options.taskTimeoutMs)
|
|
908
|
+
throw diagnosticError('INCOMPATIBLE_TIMEOUTS', 'budget.maxWallClockSeconds', this.options.taskTimeoutMs / 1000, m.budget.maxWallClockSeconds, 'Reduce the execution budget or raise the worker overall task timeout before accepting.');
|
|
909
|
+
if (data.peerIdleTimeoutMs !== undefined &&
|
|
910
|
+
(!Number.isSafeInteger(data.peerIdleTimeoutMs) || data.peerIdleTimeoutMs < 1 ||
|
|
911
|
+
(this.options.heartbeatIntervalMs ?? 15_000) >= data.peerIdleTimeoutMs))
|
|
912
|
+
throw diagnosticError('INCOMPATIBLE_TIMEOUTS', 'peerIdleTimeoutMs', { greaterThan: this.options.heartbeatIntervalMs ?? 15_000 }, data.peerIdleTimeoutMs, 'Raise the requester idle timeout or shorten the worker heartbeat interval before accepting.');
|
|
699
913
|
const contract = createContract(m, data.proposal, this.options.owner);
|
|
700
|
-
const approval = await this.approvalProvider.approve({
|
|
914
|
+
const approval = await abortable(this.approvalProvider.approve({
|
|
701
915
|
taskId: m.taskId,
|
|
702
916
|
gate: 'contract_acceptance',
|
|
703
917
|
target: contract.contractDigest,
|
|
704
918
|
details: {
|
|
705
919
|
manifest: m,
|
|
920
|
+
input: data.input,
|
|
706
921
|
capabilities: data.capabilityChains,
|
|
707
922
|
from: first.payload.senderDid,
|
|
708
923
|
},
|
|
709
924
|
signal: activeSignal,
|
|
710
|
-
});
|
|
925
|
+
}), activeSignal);
|
|
711
926
|
gateway = new OwnerGateway(this.options.owner, contract, policy, this.revocation);
|
|
712
927
|
const accepted = gateway.accept(approval);
|
|
928
|
+
saveCheckpoint({ phase: 'accepted' });
|
|
713
929
|
if (this.options.dailyLimit)
|
|
714
930
|
consumeDailyTask(this.options.dailyLimit.path, this.options.dailyLimit.maxTasks, m.taskId);
|
|
715
931
|
if (this.options.inboxPath) {
|
|
@@ -735,7 +951,6 @@ export class WorkerNode {
|
|
|
735
951
|
finally {
|
|
736
952
|
closeSync(fd);
|
|
737
953
|
}
|
|
738
|
-
this.persistedEnvelopeIds?.add(first.payload.envelopeId);
|
|
739
954
|
await acknowledgeRelay(relay, first);
|
|
740
955
|
// Notification errors must not fail a persisted task.
|
|
741
956
|
try {
|
|
@@ -748,6 +963,9 @@ export class WorkerNode {
|
|
|
748
963
|
await wire.sendEvent('ACCEPT', { contract, transition: accepted, approval });
|
|
749
964
|
this.options.onEvent?.('ACCEPT', m.taskId);
|
|
750
965
|
await wire.sendEvent('START', gateway.transition('START'));
|
|
966
|
+
heartbeat = setTimeout(() => void sendHeartbeat(), this.options.heartbeatIntervalMs ?? 15_000);
|
|
967
|
+
heartbeat.unref();
|
|
968
|
+
timeouts.running();
|
|
751
969
|
this.options.onEvent?.('IN_PROGRESS', m.taskId);
|
|
752
970
|
const extra = this.options.adapter;
|
|
753
971
|
const tools = this.options.tools ?? extra.tools ?? {}, backend = this.options.toolBackend ?? extra.toolBackend;
|
|
@@ -791,6 +1009,8 @@ export class WorkerNode {
|
|
|
791
1009
|
});
|
|
792
1010
|
});
|
|
793
1011
|
workspace = await mkdtemp(join(tmpdir(), 'acp-adapter-'));
|
|
1012
|
+
saveCheckpoint({ phase: 'running', workspace });
|
|
1013
|
+
let invocationCount = 0;
|
|
794
1014
|
const executionSignal = AbortSignal.any([
|
|
795
1015
|
activeSignal,
|
|
796
1016
|
gateway.quota.controller.signal,
|
|
@@ -895,7 +1115,15 @@ export class WorkerNode {
|
|
|
895
1115
|
signal: executionSignal,
|
|
896
1116
|
})
|
|
897
1117
|
: undefined;
|
|
898
|
-
|
|
1118
|
+
executionSignal.throwIfAborted();
|
|
1119
|
+
const invocation = { id: m.taskId + ':' + invocationCount++, tool,
|
|
1120
|
+
argsDigest: digest(args), completed: false };
|
|
1121
|
+
saveCheckpoint({ invocations: [...(checkpoint?.invocations ?? []), invocation] });
|
|
1122
|
+
const result = asJson(await proxies[index].callTool(tool, args, approval));
|
|
1123
|
+
// A backend may finish after cancellation. Leave its uncertain result for inspection.
|
|
1124
|
+
if (!executionSignal.aborted)
|
|
1125
|
+
saveCheckpoint({ invocations: (checkpoint?.invocations ?? []).map((call) => call.id === invocation.id ? { ...call, completed: true } : call) });
|
|
1126
|
+
return result;
|
|
899
1127
|
},
|
|
900
1128
|
};
|
|
901
1129
|
const output = await abortable(this.options.adapter.executeTask(context), executionSignal);
|
|
@@ -915,19 +1143,29 @@ export class WorkerNode {
|
|
|
915
1143
|
...(sandboxBoundary ? { sandboxBoundary } : {}),
|
|
916
1144
|
});
|
|
917
1145
|
const transition = gateway.transition('DELIVER');
|
|
1146
|
+
saveCheckpoint({ phase: 'delivered' });
|
|
918
1147
|
await wire.sendEvent('DELIVER', { ...delivery, transition });
|
|
919
1148
|
this.options.onEvent?.('DELIVER', m.taskId);
|
|
1149
|
+
timeouts.wait();
|
|
920
1150
|
const verification = await wire.receiveEvent(['VERIFY_ACCEPT', 'VERIFY_REJECT']);
|
|
921
1151
|
gateway.machine.apply(verification.payload.payload);
|
|
922
1152
|
if (verification.payload.eventType === 'VERIFY_REJECT')
|
|
923
1153
|
throw new Error('VERIFICATION_REJECTED');
|
|
924
1154
|
const settled = await wire.receiveEvent('SETTLED');
|
|
925
1155
|
gateway.machine.apply(settled.payload.payload);
|
|
926
|
-
const bundle = await wire.finishWorker()
|
|
1156
|
+
const bundle = await wire.finishWorker((sealed) => {
|
|
1157
|
+
saveOutcome({ code: 'TASK_SETTLED', result: { taskId: m.taskId, state: 'SETTLED', deliverable: output, bundle: sealed } });
|
|
1158
|
+
});
|
|
1159
|
+
saveCheckpoint({ notificationPending: false });
|
|
927
1160
|
this.options.onEvent?.('SETTLED', m.taskId);
|
|
928
1161
|
return { taskId: m.taskId, state: 'SETTLED', deliverable: output, bundle };
|
|
929
1162
|
}
|
|
930
1163
|
catch (e) {
|
|
1164
|
+
if (checkpoint?.outcome?.result) {
|
|
1165
|
+
// The bilateral checkpoint is already durable even if its final relay send failed.
|
|
1166
|
+
await this.notifyOutcome(checkpoint).then(() => saveCheckpoint({ notificationPending: false })).catch(() => { });
|
|
1167
|
+
return checkpoint.outcome.result;
|
|
1168
|
+
}
|
|
931
1169
|
if (gateway &&
|
|
932
1170
|
(wire.cancelHandled || cancelController.signal.aborted) &&
|
|
933
1171
|
['ACCEPTED', 'IN_PROGRESS', 'CANCELLED'].includes(gateway.machine.state)) {
|
|
@@ -938,7 +1176,10 @@ export class WorkerNode {
|
|
|
938
1176
|
throw new Error('INVALID_CANCEL');
|
|
939
1177
|
if (gateway.machine.state !== 'CANCELLED')
|
|
940
1178
|
gateway.machine.apply(body.transition);
|
|
941
|
-
const bundle = await wire.finishWorker()
|
|
1179
|
+
const bundle = await wire.finishWorker((sealed) => {
|
|
1180
|
+
saveOutcome({ code: 'TASK_CANCELLED', result: { taskId: m.taskId, state: 'CANCELLED', deliverable: null, bundle: sealed } });
|
|
1181
|
+
});
|
|
1182
|
+
saveCheckpoint({ notificationPending: false });
|
|
942
1183
|
this.options.onEvent?.('CANCELLED', m.taskId);
|
|
943
1184
|
return { taskId: m.taskId, state: 'CANCELLED', deliverable: null, bundle };
|
|
944
1185
|
}
|
|
@@ -948,10 +1189,16 @@ export class WorkerNode {
|
|
|
948
1189
|
}
|
|
949
1190
|
}
|
|
950
1191
|
await progressTail;
|
|
1192
|
+
preserveWorkspace = activeSignal.aborted || idleSignal.aborted;
|
|
1193
|
+
if (idleSignal.aborted)
|
|
1194
|
+
throw e;
|
|
1195
|
+
const diagnostic = preserveWorkspace ? { code: 'TASK_INTERRUPTED' } : rejectionBody(progressFailure ?? e);
|
|
1196
|
+
saveOutcome(diagnostic);
|
|
951
1197
|
this.reportRejection(m.taskId, progressFailure ?? e);
|
|
952
1198
|
if (!wireFailureSent) {
|
|
953
1199
|
wireFailureSent = true;
|
|
954
|
-
await wire.rawSend('ERROR',
|
|
1200
|
+
await wire.rawSend('ERROR', diagnostic)
|
|
1201
|
+
.then(() => saveCheckpoint({ notificationPending: false })).catch(() => { });
|
|
955
1202
|
}
|
|
956
1203
|
// A quota abort already surfaces as EXECUTION_ABORTED. Keep that error.
|
|
957
1204
|
// A caught INVALID_PROGRESS still fails the session with the sticky error.
|
|
@@ -961,11 +1208,32 @@ export class WorkerNode {
|
|
|
961
1208
|
throw e;
|
|
962
1209
|
}
|
|
963
1210
|
finally {
|
|
1211
|
+
heartbeatStopped = true;
|
|
1212
|
+
clearTimeout(heartbeat);
|
|
1213
|
+
timeouts.close();
|
|
1214
|
+
wire.close();
|
|
964
1215
|
if (trackedTaskId)
|
|
965
1216
|
this.liveChains.delete(trackedTaskId);
|
|
966
1217
|
gateway?.close();
|
|
967
|
-
if (
|
|
968
|
-
|
|
1218
|
+
if (idleSignal.aborted && checkpoint?.phase !== 'terminal')
|
|
1219
|
+
preserveWorkspace = true;
|
|
1220
|
+
if (workspace && !preserveWorkspace) {
|
|
1221
|
+
try {
|
|
1222
|
+
await rm(workspace, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
1223
|
+
}
|
|
1224
|
+
catch (error) {
|
|
1225
|
+
const code = error?.code ?? 'WORKSPACE_CLEANUP_FAILED';
|
|
1226
|
+
// Reporting residual files must not replace settlement or the execution error.
|
|
1227
|
+
try {
|
|
1228
|
+
if (this.options.onWorkspaceCleanupError)
|
|
1229
|
+
void Promise.resolve(this.options.onWorkspaceCleanupError({ taskId: m.taskId, workspaceDir: workspace, code }))
|
|
1230
|
+
.catch(() => { });
|
|
1231
|
+
else
|
|
1232
|
+
console.warn('Workspace cleanup incomplete (' + code + '): ' + workspace);
|
|
1233
|
+
}
|
|
1234
|
+
catch { }
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
969
1237
|
}
|
|
970
1238
|
}
|
|
971
1239
|
/**
|
|
@@ -1013,6 +1281,7 @@ export class WorkerNode {
|
|
|
1013
1281
|
}
|
|
1014
1282
|
};
|
|
1015
1283
|
return {
|
|
1284
|
+
get connectionStatus() { return relay.connectionStatus; },
|
|
1016
1285
|
send: (envelope) => relay.send(envelope),
|
|
1017
1286
|
...(relay.acknowledgeDelivery ? { acknowledgeDelivery: (did, id) => relay.acknowledgeDelivery(did, id) } : {}),
|
|
1018
1287
|
receive,
|
|
@@ -1103,30 +1372,8 @@ export class WorkerNode {
|
|
|
1103
1372
|
invite.agentDid !== this.agentCard.payload.agentDid ||
|
|
1104
1373
|
invite.relayUrl !== this.agentCard.payload.relayEndpoint)
|
|
1105
1374
|
throw new Error('INVITE_WORKER_MISMATCH');
|
|
1375
|
+
this.revocation.assertNotRevoked('invite', digest(invite));
|
|
1106
1376
|
this.revocation.assertNotRevoked('invite', token);
|
|
1107
|
-
const targetId = this.inviteDigest(token);
|
|
1108
|
-
if (targetId)
|
|
1109
|
-
this.revocation.assertNotRevoked('invite', targetId);
|
|
1110
|
-
}
|
|
1111
|
-
/**
|
|
1112
|
-
* Digest of an invite the worker already holds. Parsing does not dial the relay.
|
|
1113
|
-
* A loopback invite is hashed even when this worker did not opt into dialing it,
|
|
1114
|
-
* so a digest revocation still matches. An unparseable token is not a revocation.
|
|
1115
|
-
*/
|
|
1116
|
-
inviteDigest(token) {
|
|
1117
|
-
try {
|
|
1118
|
-
return inviteRevocationId(token, { allowPrivateRelay: this.options.allowPrivateRelay });
|
|
1119
|
-
}
|
|
1120
|
-
catch (error) {
|
|
1121
|
-
if (!(error instanceof Error) || error.message !== 'SSRF_BLOCKED_RELAY_URL')
|
|
1122
|
-
return undefined;
|
|
1123
|
-
try {
|
|
1124
|
-
return inviteRevocationId(token, { allowPrivateRelay: true });
|
|
1125
|
-
}
|
|
1126
|
-
catch {
|
|
1127
|
-
return undefined;
|
|
1128
|
-
}
|
|
1129
|
-
}
|
|
1130
1377
|
}
|
|
1131
1378
|
/**
|
|
1132
1379
|
* Before ACCEPT, every verified capability chain is checked against revocation
|
|
@@ -1309,7 +1556,7 @@ function rejectionBody(error) {
|
|
|
1309
1556
|
}
|
|
1310
1557
|
function safeError(e) {
|
|
1311
1558
|
const s = String(e);
|
|
1312
|
-
for (const code of ['CLAIM_LOST', 'DAILY_LIMIT_REACHED', 'DAILY_TASK_REPLAY', 'DAILY_LIMIT_CLOCK_ROLLBACK', 'DAILY_LIMIT_BUSY', 'DAILY_LIMIT_INVALID', 'UNENCRYPTED_ENVELOPE_REJECTED', 'INVITE_REQUIRED', 'INVITE_EXPIRED', 'INVITE_WORKER_MISMATCH', 'TASK_SCOPE_REJECTED'])
|
|
1559
|
+
for (const code of ['INCOMPATIBLE_TIMEOUTS', 'CLAIM_LOST', 'DAILY_LIMIT_REACHED', 'DAILY_TASK_REPLAY', 'DAILY_LIMIT_CLOCK_ROLLBACK', 'DAILY_LIMIT_BUSY', 'DAILY_LIMIT_INVALID', 'UNENCRYPTED_ENVELOPE_REJECTED', 'INVITE_REQUIRED', 'INVITE_EXPIRED', 'INVITE_WORKER_MISMATCH', 'TASK_SCOPE_REJECTED'])
|
|
1313
1560
|
if (s === 'Error: ' + code || s.startsWith('Error: ' + code + ':'))
|
|
1314
1561
|
return code;
|
|
1315
1562
|
if (s.includes('TOKEN_REVOKED'))
|
|
@@ -1336,6 +1583,24 @@ async function abortable(promise, signal) {
|
|
|
1336
1583
|
signal.removeEventListener('abort', listener);
|
|
1337
1584
|
}
|
|
1338
1585
|
}
|
|
1586
|
+
/** Completion failure preserves local verification without claiming a sealed checkpoint. */
|
|
1587
|
+
export class IncompleteSettlementError extends Error {
|
|
1588
|
+
incompleteReceipt;
|
|
1589
|
+
code;
|
|
1590
|
+
detail;
|
|
1591
|
+
recovery;
|
|
1592
|
+
constructor(incompleteReceipt, cause) {
|
|
1593
|
+
super(incompleteReceipt.completionError, { cause });
|
|
1594
|
+
this.incompleteReceipt = incompleteReceipt;
|
|
1595
|
+
this.name = 'IncompleteSettlementError';
|
|
1596
|
+
if (cause instanceof Error && 'code' in cause && typeof cause.code === 'string')
|
|
1597
|
+
this.code = cause.code;
|
|
1598
|
+
if (cause instanceof PeerRejectedError) {
|
|
1599
|
+
this.detail = cause.detail;
|
|
1600
|
+
this.recovery = cause.recovery;
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1339
1604
|
/**
|
|
1340
1605
|
* Queue a signed REVOKE envelope on a worker inbox.
|
|
1341
1606
|
* The payload is a SignedRevocationNotice. Encryption is on unless encrypt is false,
|
|
@@ -1378,6 +1643,7 @@ export class RequesterClient {
|
|
|
1378
1643
|
approvalProvider;
|
|
1379
1644
|
constructor(options) {
|
|
1380
1645
|
this.options = options;
|
|
1646
|
+
validateSessionTimeouts(options);
|
|
1381
1647
|
this.approvalProvider =
|
|
1382
1648
|
options.approvalProvider ?? new InteractiveCliApprovalProvider(options.owner);
|
|
1383
1649
|
}
|
|
@@ -1386,7 +1652,7 @@ export class RequesterClient {
|
|
|
1386
1652
|
* `to` is a worker DID or an acp-invite token. The notice is signed by this client.
|
|
1387
1653
|
*/
|
|
1388
1654
|
async sendRevocation(options) {
|
|
1389
|
-
const invite = options.to
|
|
1655
|
+
const invite = isInviteTarget(options.to)
|
|
1390
1656
|
? parseAndVerifyInviteToken(options.to, Date.now(), {
|
|
1391
1657
|
allowPrivateRelay: this.options.allowPrivateRelay,
|
|
1392
1658
|
})
|
|
@@ -1413,8 +1679,12 @@ export class RequesterClient {
|
|
|
1413
1679
|
});
|
|
1414
1680
|
}
|
|
1415
1681
|
async delegate(options, hooks) {
|
|
1682
|
+
const timeoutOptions = resolveSessionTimeouts(this.options, options);
|
|
1683
|
+
validateSessionTimeouts(timeoutOptions);
|
|
1684
|
+
if (timeoutOptions.executionTimeoutMs !== undefined && timeoutOptions.executionTimeoutMs < 1000)
|
|
1685
|
+
throw new Error('INVALID_TIMEOUT: executionTimeoutMs must be at least 1000 (whole-second contract budget)');
|
|
1416
1686
|
const onProgress = hooks?.onProgress ?? options.onProgress;
|
|
1417
|
-
const invite = options.to
|
|
1687
|
+
const invite = isInviteTarget(options.to)
|
|
1418
1688
|
? parseAndVerifyInviteToken(options.to, Date.now(), {
|
|
1419
1689
|
allowPrivateRelay: this.options.allowPrivateRelay,
|
|
1420
1690
|
})
|
|
@@ -1431,6 +1701,7 @@ export class RequesterClient {
|
|
|
1431
1701
|
const relay = transport(relayOption, this.options.owner, {
|
|
1432
1702
|
allowPrivateRelay: this.options.allowPrivateRelay,
|
|
1433
1703
|
lookup: this.options.lookup,
|
|
1704
|
+
requestTimeoutMs: timeoutOptions.connectTimeoutMs,
|
|
1434
1705
|
}), owner = this.options.owner, inbox = typeof relayOption === 'string'
|
|
1435
1706
|
? recipientInbox(this, relay, owner.did, relayOption.replace(/\/$/, ''))
|
|
1436
1707
|
: recipientInbox(relayOption, relay, owner.did);
|
|
@@ -1449,6 +1720,9 @@ export class RequesterClient {
|
|
|
1449
1720
|
const manifest = options.manifest ??
|
|
1450
1721
|
buildTaskManifest({
|
|
1451
1722
|
...options,
|
|
1723
|
+
taskTimeoutMs: timeoutOptions.taskTimeoutMs ?? timeoutOptions.timeoutMs,
|
|
1724
|
+
budget: { ...options.budget, ...(timeoutOptions.executionTimeoutMs !== undefined
|
|
1725
|
+
? { maxWallClockSeconds: options.budget?.maxWallClockSeconds ?? Math.floor(timeoutOptions.executionTimeoutMs / 1000) } : {}) },
|
|
1452
1726
|
ownerDid: owner.did,
|
|
1453
1727
|
workerDid: peer,
|
|
1454
1728
|
agentDid: invite?.agentDid ?? card?.payload.agentDid ?? peer,
|
|
@@ -1464,6 +1738,12 @@ export class RequesterClient {
|
|
|
1464
1738
|
manifest.budget[field] = Math.min(manifest.budget[field], limit);
|
|
1465
1739
|
}
|
|
1466
1740
|
}
|
|
1741
|
+
if (options.manifest && timeoutOptions.executionTimeoutMs !== undefined &&
|
|
1742
|
+
manifest.budget.maxWallClockSeconds * 1000 > timeoutOptions.executionTimeoutMs)
|
|
1743
|
+
throw new Error('INCOMPATIBLE_TIMEOUTS: manifest budget exceeds executionTimeoutMs');
|
|
1744
|
+
if (options.budget?.maxWallClockSeconds !== undefined && timeoutOptions.executionTimeoutMs !== undefined &&
|
|
1745
|
+
options.budget.maxWallClockSeconds * 1000 > timeoutOptions.executionTimeoutMs)
|
|
1746
|
+
throw new Error('INCOMPATIBLE_TIMEOUTS: requested budget exceeds executionTimeoutMs');
|
|
1467
1747
|
validateManifest(manifest);
|
|
1468
1748
|
if (manifest.requester.ownerDid !== owner.did ||
|
|
1469
1749
|
manifest.worker?.ownerDid !== peer ||
|
|
@@ -1482,8 +1762,9 @@ export class RequesterClient {
|
|
|
1482
1762
|
const auditPath = options.auditPath ?? this.options.auditPath;
|
|
1483
1763
|
freshAudit(auditPath);
|
|
1484
1764
|
const bridge = new AbortController();
|
|
1485
|
-
const signal = sessionSignal(Math.min(this.options.timeoutMs ?? 120000, Date.parse(manifest.deadline) - Date.now()), bridge.signal);
|
|
1486
1765
|
const mailbox = inbox.attach(manifest.taskId);
|
|
1766
|
+
const timeouts = new SessionTimeouts(timeoutOptions, manifest.deadline, bridge.signal);
|
|
1767
|
+
const signal = timeouts.signal;
|
|
1487
1768
|
const wire = new SessionWire(mailboxRelay(relay, mailbox), owner, peer, manifest.taskId, [owner.did, peer], signal, auditPath, this.options.allowUnencryptedEnvelopes === true);
|
|
1488
1769
|
if (onProgress) {
|
|
1489
1770
|
wire.onProgress = async (progress, envelope) => {
|
|
@@ -1497,6 +1778,7 @@ export class RequesterClient {
|
|
|
1497
1778
|
});
|
|
1498
1779
|
};
|
|
1499
1780
|
}
|
|
1781
|
+
let verifiedReceipt;
|
|
1500
1782
|
let gateway;
|
|
1501
1783
|
let deferredCancel = false;
|
|
1502
1784
|
let cancelStarted = false;
|
|
@@ -1549,6 +1831,7 @@ export class RequesterClient {
|
|
|
1549
1831
|
proposal: signPayload(manifest, owner),
|
|
1550
1832
|
input: options.input,
|
|
1551
1833
|
capabilityChains,
|
|
1834
|
+
peerIdleTimeoutMs: timeoutOptions.transportIdleTimeoutMs ?? 90_000,
|
|
1552
1835
|
...(invite ? { inviteToken: options.to } : {}),
|
|
1553
1836
|
});
|
|
1554
1837
|
const accept = (await wire.receiveEvent('ACCEPT')).payload.payload;
|
|
@@ -1561,8 +1844,13 @@ export class RequesterClient {
|
|
|
1561
1844
|
gateway.machine.apply((await wire.receiveEvent('START')).payload.payload);
|
|
1562
1845
|
if (options.signal?.aborted || deferredCancel)
|
|
1563
1846
|
startCancel();
|
|
1847
|
+
timeouts.executing(timeoutOptions.executionTimeoutMs ?? manifest.budget.maxWallClockSeconds * 1000);
|
|
1848
|
+
wire.onActivity = () => timeouts.activity();
|
|
1564
1849
|
const envelope = await wire.receiveEvent('DELIVER'), data = envelope.payload.payload;
|
|
1850
|
+
wire.onActivity = undefined;
|
|
1851
|
+
timeouts.wait();
|
|
1565
1852
|
gateway.machine.apply(data.transition);
|
|
1853
|
+
hooks?.onDelivered?.();
|
|
1566
1854
|
const approval = () => this.approvalProvider.approve({
|
|
1567
1855
|
taskId: manifest.taskId,
|
|
1568
1856
|
gate: 'final_verification',
|
|
@@ -1570,11 +1858,21 @@ export class RequesterClient {
|
|
|
1570
1858
|
details: { manifest, deliverableDigest: data.artifact.payload.digest },
|
|
1571
1859
|
signal,
|
|
1572
1860
|
});
|
|
1573
|
-
const { result, verification, settlement } = await gateway.verifyAndSettle(wire.sealed(envelope), this.options.verificationEngine ?? new VerificationEngine(), approval, { allowUnencryptedEnvelopes: this.options.allowUnencryptedEnvelopes === true });
|
|
1861
|
+
const { result, verification, settlement } = await abortable(gateway.verifyAndSettle(wire.sealed(envelope), this.options.verificationEngine ?? new VerificationEngine(), approval, { allowUnencryptedEnvelopes: this.options.allowUnencryptedEnvelopes === true }), signal);
|
|
1574
1862
|
if (!result.accepted || !settlement) {
|
|
1575
1863
|
await wire.sendEvent('VERIFY_REJECT', verification);
|
|
1576
1864
|
throw new Error('VERIFICATION_REJECTED: ' + result.reason);
|
|
1577
1865
|
}
|
|
1866
|
+
verifiedReceipt = {
|
|
1867
|
+
workerDid: manifest.worker.agentDid,
|
|
1868
|
+
taskId: manifest.taskId,
|
|
1869
|
+
outputDigest: digest(data.artifact.payload.data),
|
|
1870
|
+
deliverable: data.artifact.payload.data,
|
|
1871
|
+
accepted: true,
|
|
1872
|
+
checkpointComplete: false,
|
|
1873
|
+
delivery: wire.sealed(envelope),
|
|
1874
|
+
verification: result,
|
|
1875
|
+
};
|
|
1578
1876
|
await wire.sendEvent('VERIFY_ACCEPT', verification);
|
|
1579
1877
|
await wire.sendEvent('SETTLED', settlement);
|
|
1580
1878
|
return {
|
|
@@ -1586,23 +1884,69 @@ export class RequesterClient {
|
|
|
1586
1884
|
};
|
|
1587
1885
|
}
|
|
1588
1886
|
catch (e) {
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1887
|
+
try {
|
|
1888
|
+
if (e instanceof PeerRejectedError && e.recovery) {
|
|
1889
|
+
if (e.recovery.proposalDigest !== digest({ manifest, input: options.input }))
|
|
1890
|
+
throw new Error('INVALID_RECOVERY_OUTCOME');
|
|
1891
|
+
const cached = e.recovery.result;
|
|
1892
|
+
if (cached) {
|
|
1893
|
+
const proposed = cached.bundle?.entries?.find((entry) => entry.body.eventType === 'PROPOSE')
|
|
1894
|
+
?.body.payload;
|
|
1895
|
+
const accepted = cached.bundle?.entries?.find((entry) => entry.body.eventType === 'ACCEPT')
|
|
1896
|
+
?.body.payload;
|
|
1897
|
+
const delivered = cached.bundle?.entries?.find((entry) => entry.body.eventType === 'DELIVER')
|
|
1898
|
+
?.body.payload;
|
|
1899
|
+
if (cached.taskId !== manifest.taskId || !['SETTLED', 'CANCELLED'].includes(cached.state) ||
|
|
1900
|
+
!verifyAudit(cached.bundle, { taskId: manifest.taskId, actors: [owner.did, peer] }).valid ||
|
|
1901
|
+
!proposed || digest({ manifest: proposed.proposal.payload, input: proposed.input }) !== e.recovery.proposalDigest ||
|
|
1902
|
+
!accepted || !verifyContract(accepted.contract) || digest(accepted.contract.manifest) !== digest(manifest) ||
|
|
1903
|
+
(cached.state === 'CANCELLED' && (e.code !== 'TASK_CANCELLED' || cached.deliverable !== null ||
|
|
1904
|
+
cached.bundle.entries.at(-1)?.body.eventType !== 'CANCEL')) ||
|
|
1905
|
+
(cached.state === 'SETTLED' && (!delivered ||
|
|
1906
|
+
e.code !== 'TASK_SETTLED' ||
|
|
1907
|
+
digest(delivered.artifact.payload.data) !== digest(cached.deliverable) ||
|
|
1908
|
+
cached.bundle.entries.at(-1)?.body.eventType !== 'SETTLED')))
|
|
1909
|
+
throw new Error('INVALID_RECOVERY_OUTCOME');
|
|
1910
|
+
if (cached.state === 'SETTLED')
|
|
1911
|
+
hooks?.onDelivered?.();
|
|
1912
|
+
return cached;
|
|
1913
|
+
}
|
|
1594
1914
|
}
|
|
1595
|
-
|
|
1596
|
-
|
|
1915
|
+
if (verifiedReceipt)
|
|
1916
|
+
throw e;
|
|
1917
|
+
// A closed recipient inbox is terminal for every attached task. Do not wait
|
|
1918
|
+
// for another relay send (or cancellation handshake) on the failed connection.
|
|
1919
|
+
if (e instanceof Error && 'code' in e && ['RELAY_TERMINAL', 'RELAY_ENVELOPE_EXPIRED'].includes(String(e.code)))
|
|
1920
|
+
throw e;
|
|
1921
|
+
if (cancelStarted) {
|
|
1922
|
+
try {
|
|
1923
|
+
await wire.cancelSession();
|
|
1924
|
+
const bundle = await wire.finishRequester();
|
|
1925
|
+
throw Object.assign(new Error('CANCELLED'), { bundle });
|
|
1926
|
+
}
|
|
1927
|
+
catch (inner) {
|
|
1928
|
+
if (inner instanceof Error && inner.message === 'CANCELLED')
|
|
1929
|
+
throw inner;
|
|
1930
|
+
await wire.rawSend('ERROR', { code: safeError(inner) }).catch(() => { });
|
|
1597
1931
|
throw inner;
|
|
1598
|
-
|
|
1599
|
-
throw inner;
|
|
1932
|
+
}
|
|
1600
1933
|
}
|
|
1934
|
+
await wire.rawSend('ERROR', { code: safeError(e) }).catch(() => { });
|
|
1935
|
+
throw e;
|
|
1936
|
+
}
|
|
1937
|
+
catch (error) {
|
|
1938
|
+
if (!verifiedReceipt)
|
|
1939
|
+
throw error;
|
|
1940
|
+
throw new IncompleteSettlementError({
|
|
1941
|
+
...verifiedReceipt,
|
|
1942
|
+
entries: wire.entries.slice(),
|
|
1943
|
+
completionError: error instanceof Error ? error.message : String(error),
|
|
1944
|
+
}, error);
|
|
1601
1945
|
}
|
|
1602
|
-
await wire.rawSend('ERROR', { code: safeError(e) }).catch(() => { });
|
|
1603
|
-
throw e;
|
|
1604
1946
|
}
|
|
1605
1947
|
finally {
|
|
1948
|
+
timeouts.close();
|
|
1949
|
+
wire.close();
|
|
1606
1950
|
options.signal?.removeEventListener('abort', onUserAbort);
|
|
1607
1951
|
inbox.detach(manifest.taskId);
|
|
1608
1952
|
gateway?.close();
|
|
@@ -1613,6 +1957,10 @@ export class RequesterClient {
|
|
|
1613
1957
|
throw new Error('WORKERS_REQUIRED');
|
|
1614
1958
|
if (options.mode !== 'race' && options.mode !== 'quorum')
|
|
1615
1959
|
throw new Error('INVALID_DELEGATION_MODE');
|
|
1960
|
+
const completionTimeoutMs = options.completionTimeoutMs ?? 45_000;
|
|
1961
|
+
assertPositive('COMPLETION_TIMEOUT', completionTimeoutMs);
|
|
1962
|
+
if (completionTimeoutMs > 2_147_483_647)
|
|
1963
|
+
throw new Error('INVALID_COMPLETION_TIMEOUT');
|
|
1616
1964
|
const count = options.workers.length;
|
|
1617
1965
|
const quorumSize = options.quorumSize ?? Math.floor(count / 2) + 1;
|
|
1618
1966
|
if (options.mode === 'quorum' &&
|
|
@@ -1633,13 +1981,23 @@ export class RequesterClient {
|
|
|
1633
1981
|
}
|
|
1634
1982
|
const controllers = options.workers.map(() => new AbortController());
|
|
1635
1983
|
const finished = options.workers.map(() => false);
|
|
1984
|
+
const delivered = options.workers.map(() => false);
|
|
1985
|
+
const completionTimers = [];
|
|
1636
1986
|
const receipts = [];
|
|
1987
|
+
const incompleteReceipts = [];
|
|
1637
1988
|
let failures = 0;
|
|
1638
1989
|
let winnerDigest;
|
|
1639
1990
|
const abortUnfinished = (reason) => {
|
|
1640
1991
|
controllers.forEach((controller, index) => {
|
|
1641
|
-
if (
|
|
1992
|
+
if (finished[index])
|
|
1993
|
+
return;
|
|
1994
|
+
if (!delivered[index])
|
|
1642
1995
|
controller.abort(reason);
|
|
1996
|
+
else if (!completionTimers[index]) {
|
|
1997
|
+
// DELIVER has left execution: preserve verification and bilateral checkpointing.
|
|
1998
|
+
completionTimers[index] = setTimeout(() => controller.abort(new Error('COMPLETION_TIMEOUT')), completionTimeoutMs);
|
|
1999
|
+
completionTimers[index].unref();
|
|
2000
|
+
}
|
|
1643
2001
|
});
|
|
1644
2002
|
};
|
|
1645
2003
|
const jobs = resolved.map(async (worker, index) => {
|
|
@@ -1654,6 +2012,13 @@ export class RequesterClient {
|
|
|
1654
2012
|
description: options.description,
|
|
1655
2013
|
input: options.input,
|
|
1656
2014
|
outputSchema: options.outputSchema,
|
|
2015
|
+
executionTimeoutMs: options.executionTimeoutMs,
|
|
2016
|
+
timeoutMs: options.timeoutMs,
|
|
2017
|
+
taskTimeoutMs: options.taskTimeoutMs,
|
|
2018
|
+
waitTimeoutMs: options.waitTimeoutMs,
|
|
2019
|
+
connectTimeoutMs: options.connectTimeoutMs,
|
|
2020
|
+
transportIdleTimeoutMs: options.transportIdleTimeoutMs,
|
|
2021
|
+
heartbeatIntervalMs: options.heartbeatIntervalMs,
|
|
1657
2022
|
budget: options.budget,
|
|
1658
2023
|
capabilities: options.capabilities,
|
|
1659
2024
|
minimumIsolationLevel: options.minimumIsolationLevel,
|
|
@@ -1662,7 +2027,7 @@ export class RequesterClient {
|
|
|
1662
2027
|
auditPath: options.auditPath
|
|
1663
2028
|
? options.auditPath.replace(/(\.jsonl)?$/, '.' + index + '.jsonl')
|
|
1664
2029
|
: undefined,
|
|
1665
|
-
});
|
|
2030
|
+
}, { onDelivered: () => { delivered[index] = true; } });
|
|
1666
2031
|
finished[index] = true;
|
|
1667
2032
|
// SETTLED always carries a verified deliverable, including JSON null.
|
|
1668
2033
|
if (result.state !== 'SETTLED')
|
|
@@ -1689,12 +2054,17 @@ export class RequesterClient {
|
|
|
1689
2054
|
return { index, result, outputDigest, workerDid };
|
|
1690
2055
|
}
|
|
1691
2056
|
catch (error) {
|
|
2057
|
+
if (error instanceof IncompleteSettlementError)
|
|
2058
|
+
incompleteReceipts.push(error.incompleteReceipt);
|
|
1692
2059
|
finished[index] = true;
|
|
1693
2060
|
failures += 1;
|
|
1694
2061
|
if (options.mode === 'quorum' && !winnerDigest && failures > count - quorumSize)
|
|
1695
2062
|
abortUnfinished('QUORUM_UNREACHABLE');
|
|
1696
2063
|
throw error;
|
|
1697
2064
|
}
|
|
2065
|
+
finally {
|
|
2066
|
+
clearTimeout(completionTimers[index]);
|
|
2067
|
+
}
|
|
1698
2068
|
});
|
|
1699
2069
|
if (options.mode === 'race') {
|
|
1700
2070
|
let winner;
|
|
@@ -1702,7 +2072,7 @@ export class RequesterClient {
|
|
|
1702
2072
|
winner = await Promise.any(jobs);
|
|
1703
2073
|
}
|
|
1704
2074
|
catch {
|
|
1705
|
-
throw new Error('RACE_FAILED');
|
|
2075
|
+
throw Object.assign(new Error('RACE_FAILED'), { receipts, incompleteReceipts });
|
|
1706
2076
|
}
|
|
1707
2077
|
abortUnfinished('RACE_LOST');
|
|
1708
2078
|
await Promise.allSettled(jobs);
|
|
@@ -1721,14 +2091,15 @@ export class RequesterClient {
|
|
|
1721
2091
|
},
|
|
1722
2092
|
],
|
|
1723
2093
|
receipts,
|
|
2094
|
+
incompleteReceipts,
|
|
1724
2095
|
cancelled: resolved
|
|
1725
|
-
.filter((_, index) => index !== winner.index && controllers[index].signal.aborted)
|
|
2096
|
+
.filter((_, index) => index !== winner.index && controllers[index].signal.aborted && !delivered[index])
|
|
1726
2097
|
.map((worker) => worker.workerDid),
|
|
1727
2098
|
};
|
|
1728
2099
|
}
|
|
1729
2100
|
await Promise.allSettled(jobs);
|
|
1730
2101
|
if (!winnerDigest)
|
|
1731
|
-
throw Object.assign(new Error('QUORUM_NOT_REACHED'), { receipts });
|
|
2102
|
+
throw Object.assign(new Error('QUORUM_NOT_REACHED'), { receipts, incompleteReceipts });
|
|
1732
2103
|
const seenWinners = new Set();
|
|
1733
2104
|
const winners = receipts.filter((receipt) => {
|
|
1734
2105
|
if (receipt.outputDigest !== winnerDigest || seenWinners.has(receipt.workerDid))
|
|
@@ -1748,8 +2119,9 @@ export class RequesterClient {
|
|
|
1748
2119
|
outputDigest: receipt.outputDigest,
|
|
1749
2120
|
})),
|
|
1750
2121
|
receipts,
|
|
2122
|
+
incompleteReceipts,
|
|
1751
2123
|
cancelled: resolved
|
|
1752
|
-
.filter((_, index) => controllers[index].signal.aborted)
|
|
2124
|
+
.filter((_, index) => controllers[index].signal.aborted && !delivered[index])
|
|
1753
2125
|
.map((worker) => worker.workerDid),
|
|
1754
2126
|
};
|
|
1755
2127
|
}
|
|
@@ -1835,7 +2207,7 @@ async function canonicalizeDiscoveredWorkers(targets, options) {
|
|
|
1835
2207
|
}));
|
|
1836
2208
|
}
|
|
1837
2209
|
export function buildTaskManifest(options) {
|
|
1838
|
-
const createdAt = new Date().toISOString(), deadline = new Date(Date.now() + 600000).toISOString();
|
|
2210
|
+
const createdAt = new Date().toISOString(), deadline = new Date(Date.now() + (options.taskTimeoutMs ?? 600000)).toISOString();
|
|
1839
2211
|
return {
|
|
1840
2212
|
protocolVersion: '0.1.0',
|
|
1841
2213
|
taskId: 'task_' + randomUUID().replaceAll('-', ''),
|