reactor-effect-client 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -2
- package/dist/_internal/queue.d.ts +6 -0
- package/dist/_internal/queue.d.ts.map +1 -0
- package/dist/_internal/queue.js +30 -0
- package/dist/_internal/queue.js.map +1 -0
- package/dist/duration.d.ts +2 -0
- package/dist/duration.d.ts.map +1 -1
- package/dist/duration.js +1 -1
- package/dist/duration.js.map +1 -1
- package/dist/host.d.ts +1 -0
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +1 -0
- package/dist/host.js.map +1 -1
- package/dist/observation.d.ts +3 -0
- package/dist/observation.d.ts.map +1 -1
- package/dist/observation.js +9 -4
- package/dist/observation.js.map +1 -1
- package/dist/orchestration/h3-source.d.ts.map +1 -1
- package/dist/orchestration/h3-source.js +28 -9
- package/dist/orchestration/h3-source.js.map +1 -1
- package/dist/orchestration/index.d.ts +6 -6
- package/dist/orchestration/index.d.ts.map +1 -1
- package/dist/orchestration/index.js +3 -3
- package/dist/orchestration/index.js.map +1 -1
- package/dist/orchestration/media-buffer.d.ts +2 -1
- package/dist/orchestration/media-buffer.d.ts.map +1 -1
- package/dist/orchestration/media-buffer.js +5 -4
- package/dist/orchestration/media-buffer.js.map +1 -1
- package/dist/orchestration/queries.d.ts.map +1 -1
- package/dist/orchestration/queries.js +3 -0
- package/dist/orchestration/queries.js.map +1 -1
- package/dist/orchestration/renewal-state.d.ts +28 -4
- package/dist/orchestration/renewal-state.d.ts.map +1 -1
- package/dist/orchestration/renewal-state.js +42 -7
- package/dist/orchestration/renewal-state.js.map +1 -1
- package/dist/orchestration/renewal.d.ts +39 -3
- package/dist/orchestration/renewal.d.ts.map +1 -1
- package/dist/orchestration/renewal.js +490 -103
- package/dist/orchestration/renewal.js.map +1 -1
- package/dist/orchestration/retention.d.ts +100 -0
- package/dist/orchestration/retention.d.ts.map +1 -0
- package/dist/orchestration/retention.js +134 -0
- package/dist/orchestration/retention.js.map +1 -0
- package/dist/orchestration/scheduler-key.d.ts +13 -0
- package/dist/orchestration/scheduler-key.d.ts.map +1 -0
- package/dist/orchestration/scheduler-key.js +62 -0
- package/dist/orchestration/scheduler-key.js.map +1 -0
- package/dist/orchestration/scheduler-policy.d.ts +81 -0
- package/dist/orchestration/scheduler-policy.d.ts.map +1 -0
- package/dist/orchestration/scheduler-policy.js +200 -0
- package/dist/orchestration/scheduler-policy.js.map +1 -0
- package/dist/orchestration/scheduler.d.ts +184 -0
- package/dist/orchestration/scheduler.d.ts.map +1 -0
- package/dist/orchestration/scheduler.js +1680 -0
- package/dist/orchestration/scheduler.js.map +1 -0
- package/dist/orchestration/source-slot.d.ts +20 -5
- package/dist/orchestration/source-slot.d.ts.map +1 -1
- package/dist/orchestration/source-slot.js +151 -29
- package/dist/orchestration/source-slot.js.map +1 -1
- package/dist/orchestration/types.d.ts +545 -6
- package/dist/orchestration/types.d.ts.map +1 -1
- package/dist/orchestration/types.js +105 -1
- package/dist/orchestration/types.js.map +1 -1
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +2 -1
- package/dist/session.js.map +1 -1
- package/dist/simulation/_internal/source.d.ts.map +1 -1
- package/dist/simulation/_internal/source.js +43 -31
- package/dist/simulation/_internal/source.js.map +1 -1
- package/package.json +2 -2
- package/dist/orchestration/lineup.d.ts +0 -109
- package/dist/orchestration/lineup.d.ts.map +0 -1
- package/dist/orchestration/lineup.js +0 -249
- package/dist/orchestration/lineup.js.map +0 -1
|
@@ -21,19 +21,62 @@ import * as Sequence from "../Sequence.js";
|
|
|
21
21
|
import * as Submission from "../Submission.js";
|
|
22
22
|
import * as MediaBuffer from "./media-buffer.js";
|
|
23
23
|
import * as SourceSlot from "./source-slot.js";
|
|
24
|
+
import * as Retention from "./retention.js";
|
|
24
25
|
import { monotonicMillis } from "./elapsed.js";
|
|
25
|
-
import {
|
|
26
|
+
import { handoffEvidence, decideRenewal } from "./renewal-state.js";
|
|
26
27
|
import { PolicyFailure, captureRequest } from "./request.js";
|
|
27
28
|
import { emptyState, isIdle } from "./queries.js";
|
|
28
29
|
import { activeIds, generation, resolve } from "./routing.js";
|
|
29
30
|
import { handleContext } from "./types.js";
|
|
30
31
|
const engineOnly = (event) => event._tag === "Engine" ? Result.succeed(event.event) : Result.fail(event);
|
|
31
32
|
const renewalsOnly = (event) => event._tag === "Renewal" ? Result.succeed(event.event) : Result.fail(event);
|
|
33
|
+
/** A physical source supplies facts; the renewing handle assigns their owner. */
|
|
34
|
+
const withSessionId = (state, sessionId) => {
|
|
35
|
+
const record = (clip) => Object.freeze({ ...clip, sessionId });
|
|
36
|
+
return Object.freeze({
|
|
37
|
+
...state,
|
|
38
|
+
queued: state.queued.map(record),
|
|
39
|
+
building: Option.map(state.building, (build) => ({ ...build, record: record(build.record) })),
|
|
40
|
+
ready: state.ready.map(record),
|
|
41
|
+
playing: Option.map(state.playing, (playing) => ({
|
|
42
|
+
...playing,
|
|
43
|
+
record: Option.map(playing.record, record),
|
|
44
|
+
})),
|
|
45
|
+
});
|
|
46
|
+
};
|
|
47
|
+
/** A completed caller handle keeps only its Exit, not the routing or source closures. */
|
|
48
|
+
const logicalSubmission = (id) => {
|
|
49
|
+
let execute;
|
|
50
|
+
let inspect;
|
|
51
|
+
let outcome;
|
|
52
|
+
return {
|
|
53
|
+
attach: (submit, state) => {
|
|
54
|
+
execute = submit;
|
|
55
|
+
inspect = state;
|
|
56
|
+
},
|
|
57
|
+
complete: (exit) => {
|
|
58
|
+
outcome ??= exit;
|
|
59
|
+
execute = undefined;
|
|
60
|
+
inspect = undefined;
|
|
61
|
+
},
|
|
62
|
+
outcome: () => outcome,
|
|
63
|
+
submission: {
|
|
64
|
+
id,
|
|
65
|
+
submit: Effect.suspend(() => outcome ?? execute ?? Effect.die(new Error("Logical submission has no execution"))),
|
|
66
|
+
state: Effect.suspend(() => outcome === undefined
|
|
67
|
+
? (inspect ?? Effect.succeed({ _tag: "Prepared" }))
|
|
68
|
+
: Effect.succeed({
|
|
69
|
+
_tag: "Completed",
|
|
70
|
+
exit: outcome,
|
|
71
|
+
})),
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
};
|
|
32
75
|
/**
|
|
33
76
|
* Explicit application policy for renewal, sequence affinity and continuous
|
|
34
77
|
* recovering media. A physical source still owns each connection generation.
|
|
35
78
|
*/
|
|
36
|
-
|
|
79
|
+
const makeOwner = (options, continuous) => Effect.gen(function* () {
|
|
37
80
|
const scope = yield* Effect.scope;
|
|
38
81
|
const context = yield* Effect.context();
|
|
39
82
|
const clock = yield* Clock.Clock;
|
|
@@ -42,7 +85,6 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
42
85
|
.pipe(Effect.mapError((cause) => errorOf(cause, "InvalidState")));
|
|
43
86
|
const namespace = Array.from(random, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
44
87
|
const commands = yield* Semaphore.make(1);
|
|
45
|
-
const closeGate = yield* Semaphore.make(1);
|
|
46
88
|
const affinity = yield* Sequence.makeAffinity().pipe(Effect.mapError((cause) => errorOf(cause, "InvalidInput")));
|
|
47
89
|
// One ordered stream: engine events, renewals and media transitions.
|
|
48
90
|
const observations = new Observations();
|
|
@@ -51,25 +93,58 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
51
93
|
audioSamples: positiveLimit(options.maxQueuedAudioSamples ?? MediaBuffer.defaultLimits.audioSamples, "orchestration maxQueuedAudioSamples"),
|
|
52
94
|
})));
|
|
53
95
|
const fatal = yield* Deferred.make();
|
|
96
|
+
let renewing = true;
|
|
54
97
|
const slots = new Map();
|
|
98
|
+
const physicalOwners = new Map();
|
|
99
|
+
const retention = continuous ? yield* Retention.make(options) : undefined;
|
|
100
|
+
let incarnation = 0n;
|
|
55
101
|
const cleanups = [];
|
|
56
102
|
const seenCleanups = new Set();
|
|
57
|
-
const maxSessions = options.maxSessions ?? 64;
|
|
58
|
-
if (
|
|
103
|
+
const maxSessions = options.maxSessions ?? (continuous ? Infinity : 64);
|
|
104
|
+
if ((options.maxSessions !== undefined || !continuous) &&
|
|
105
|
+
(!Number.isSafeInteger(maxSessions) || maxSessions < 1 || maxSessions > 4096)) {
|
|
59
106
|
return yield* ReactorError.fromCode("InvalidInput", "Invalid orchestration bounds");
|
|
60
107
|
}
|
|
61
108
|
const reconnectTimeout = Duration.toMillis(yield* parsed(() => duration(options.reconnectTimeout ?? "10 seconds", "orchestration reconnectTimeout")));
|
|
109
|
+
const handoffGraceMs = Duration.toMillis(yield* parsed(() => duration(options.handoffGrace ?? "250 millis", "orchestration handoffGrace", {
|
|
110
|
+
allowZero: true,
|
|
111
|
+
// Starvation during the grace is not reported, so it stays short.
|
|
112
|
+
maximum: "5 seconds",
|
|
113
|
+
})));
|
|
62
114
|
const leadSeconds = Duration.toSeconds(yield* parsed(() => duration(options.lead ?? "30 seconds", "orchestration lead", { allowZero: true })));
|
|
63
115
|
let current;
|
|
64
116
|
let replacement = { _tag: "Absent" };
|
|
65
|
-
let opened =
|
|
117
|
+
let opened = 0n;
|
|
66
118
|
let openFailures = 0;
|
|
67
119
|
let retryAt = 0;
|
|
68
120
|
let autoplay = true;
|
|
69
121
|
let closing = false;
|
|
70
122
|
let finalReport;
|
|
123
|
+
let finalSummary;
|
|
124
|
+
let firstCleanup;
|
|
125
|
+
let initializing = true;
|
|
126
|
+
// Close reports each ownership attempt's failures once. They are recorded
|
|
127
|
+
// where the attempt and its retirement end: every waiter of a cached close
|
|
128
|
+
// sees its own Cause object, and cancellation by close is not a failure.
|
|
129
|
+
const attemptFailures = new Map();
|
|
130
|
+
const recordFailure = (attempt, reasons) => {
|
|
131
|
+
const failed = reasons.filter((reason) => !Cause.isInterruptReason(reason));
|
|
132
|
+
if (failed.length > 0)
|
|
133
|
+
attemptFailures.set(attempt, [...(attemptFailures.get(attempt) ?? []), ...failed]);
|
|
134
|
+
};
|
|
135
|
+
// Every acquisition runs on a fiber this owner forks into its scope, never
|
|
136
|
+
// on a caller's fiber. Close interrupts and joins the registered ones before
|
|
137
|
+
// retiring owned slots, so no source opens after close and every attempt's
|
|
138
|
+
// cleanup is reported, without waiting on a fiber that may be waiting on it.
|
|
139
|
+
const acquisitions = new Set();
|
|
140
|
+
const admissionClosed = () => ReactorError.fromCode("Closed", "Orchestration no longer admits sources");
|
|
141
|
+
// The owner stops renewing when close begins or its scope starts closing,
|
|
142
|
+
// whichever is first: that scope's finalizers can stop an attempt, or wait
|
|
143
|
+
// on an application, before close runs. Commands stay admitted until close
|
|
144
|
+
// begins, so an application finalizer can still issue its last commands.
|
|
145
|
+
const stopping = () => closing || scope.state._tag === "Closed";
|
|
71
146
|
let mediaState = { _tag: "Closed" };
|
|
72
|
-
let
|
|
147
|
+
let terminal;
|
|
73
148
|
let submissionSequence = 0n;
|
|
74
149
|
const announce = (event) => Effect.sync(() => observations.emit({ _tag: "Renewal", event }, 256));
|
|
75
150
|
const log = (message) => Effect.logDebug(message).pipe(Effect.annotateLogs({ module: "reactor.orchestration" }));
|
|
@@ -84,20 +159,33 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
84
159
|
? undefined
|
|
85
160
|
: yield* observations.subscribe({ capacity: 4096 }).pipe(Effect.flatMap((stream) => stream.pipe(Stream.filterMap(renewalsOnly), Stream.runForEach((event) => Effect.suspend(() => onRenewal(event)).pipe(Effect.catchCause((cause) => Effect.logWarning("onRenewal failed", cause).pipe(Effect.annotateLogs({ module: "reactor.orchestration" }))))), Effect.catchCause((cause) => Effect.logWarning("onRenewal reader stopped", cause).pipe(Effect.annotateLogs({ module: "reactor.orchestration" }))), Effect.forkIn(scope))));
|
|
86
161
|
const fail = (cause) => Effect.suspend(() => {
|
|
87
|
-
if (
|
|
162
|
+
if (terminal !== undefined)
|
|
88
163
|
return Effect.void;
|
|
89
|
-
|
|
164
|
+
terminal = Exit.succeed(cause);
|
|
90
165
|
// Publish the failed state and queues before waking a failure waiter.
|
|
91
166
|
setMedia({ _tag: "Failed", cause });
|
|
92
167
|
emit({ _tag: "SessionFailed", failure: cause });
|
|
93
|
-
buffer.fail(cause);
|
|
168
|
+
buffer.failCause(Cause.fail(cause));
|
|
94
169
|
Deferred.doneUnsafe(fatal, Effect.succeed(cause));
|
|
95
170
|
return announce({ _tag: "Failed", reason: cause.message });
|
|
96
171
|
});
|
|
172
|
+
// A failed worker must wake observers as well as failure waiters: the
|
|
173
|
+
// scheduler consumes observations, and cannot recover a vanished worker.
|
|
174
|
+
// Claim and publish synchronously before a waiter can close this owner.
|
|
175
|
+
const supervise = (effect) => effect.pipe(Effect.onExit((exit) => Effect.sync(() => {
|
|
176
|
+
if (Exit.isSuccess(exit) ||
|
|
177
|
+
terminal !== undefined ||
|
|
178
|
+
(stopping() && Cause.hasInterruptsOnly(exit.cause)))
|
|
179
|
+
return;
|
|
180
|
+
terminal = Exit.failCause(exit.cause);
|
|
181
|
+
buffer.failCause(exit.cause);
|
|
182
|
+
observations.failCause(exit.cause);
|
|
183
|
+
Deferred.doneUnsafe(fatal, terminal);
|
|
184
|
+
})));
|
|
97
185
|
// Every asynchronous activation uses this final fence. Closed/failed state
|
|
98
186
|
// cannot be replaced by a reconnect or autoplay operation that finished late.
|
|
99
187
|
const publishReady = (slot) => {
|
|
100
|
-
if (
|
|
188
|
+
if (stopping() || slot.closed || slot !== current || terminal !== undefined)
|
|
101
189
|
return false;
|
|
102
190
|
setMedia({ _tag: "Ready", sessionId: slot.source.id, generation: slot.media.generation });
|
|
103
191
|
return true;
|
|
@@ -108,7 +196,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
108
196
|
seenCleanups.add(cleanup.lease);
|
|
109
197
|
cleanups.push(cleanup);
|
|
110
198
|
};
|
|
111
|
-
const openSequences = (slot) => affinity.snapshots.pipe(Effect.map((entries) => entries.some((entry) => entry.owner === slot.
|
|
199
|
+
const openSequences = (slot) => affinity.snapshots.pipe(Effect.map((entries) => entries.some((entry) => entry.owner === slot.owner && entry.status === "open")));
|
|
112
200
|
const expired = (slot) => slot.expired();
|
|
113
201
|
const recoveryBudget = (slot) => slot.recoveryBudget();
|
|
114
202
|
// The logical output's loss: every former owner's loss while it fed the
|
|
@@ -116,18 +204,23 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
116
204
|
// replacement's loss before it fed the output is not the output's.
|
|
117
205
|
let outputLoss = SourceSlot.noLoss;
|
|
118
206
|
let ownerBaseline = SourceSlot.noLoss;
|
|
207
|
+
let outputOwner;
|
|
119
208
|
const ownerLoss = (slot) => Effect.result(slot.pressure.pipe(Effect.timeout(recoveryBudget(slot)))).pipe(Effect.map(SourceSlot.lossOf));
|
|
120
209
|
const activateOwner = (slot) => ownerLoss(slot).pipe(Effect.map((loss) => {
|
|
121
210
|
ownerBaseline = loss;
|
|
211
|
+
outputOwner = slot.owner;
|
|
122
212
|
}));
|
|
123
|
-
const retireOwner = (slot) =>
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
213
|
+
const retireOwner = (slot) => Effect.suspend(() => outputOwner !== slot.owner
|
|
214
|
+
? Effect.void
|
|
215
|
+
: ownerLoss(slot).pipe(Effect.map((loss) => {
|
|
216
|
+
outputLoss = SourceSlot.addLoss(outputLoss, SourceSlot.subtractLoss(loss, ownerBaseline));
|
|
217
|
+
ownerBaseline = SourceSlot.noLoss;
|
|
218
|
+
outputOwner = undefined;
|
|
219
|
+
})));
|
|
127
220
|
const retired = (slot) => slot.retired(buffer.forwarded);
|
|
128
221
|
const closeSlot = (slot) => slot.close;
|
|
129
222
|
const replace = (slot, cause) => Effect.gen(function* () {
|
|
130
|
-
if (slot.closed ||
|
|
223
|
+
if (slot.closed || stopping() || terminal !== undefined)
|
|
131
224
|
return;
|
|
132
225
|
const state = yield* slot.source.state;
|
|
133
226
|
const lost = activeIds(state);
|
|
@@ -150,15 +243,22 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
150
243
|
});
|
|
151
244
|
if (replacement._tag === "Ready" && replacement.slot === slot)
|
|
152
245
|
replacement = { _tag: "Absent" };
|
|
153
|
-
|
|
246
|
+
// Close retires what remains; a replacement it refuses is no failure.
|
|
247
|
+
if (slot !== current || stopping())
|
|
154
248
|
return;
|
|
155
249
|
const pending = replacement;
|
|
250
|
+
if (!renewing && pending._tag === "Absent") {
|
|
251
|
+
yield* fail(cause);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
156
254
|
const selected = pending._tag === "Ready"
|
|
157
255
|
? pending.slot
|
|
158
|
-
: yield* pending._tag === "Opening"
|
|
256
|
+
: yield* pending._tag === "Opening"
|
|
257
|
+
? Effect.uninterruptibleMask((restore) => joinAcquisition(pending.fiber, restore))
|
|
258
|
+
: acquire;
|
|
159
259
|
replacement = { _tag: "Absent" };
|
|
160
|
-
if (
|
|
161
|
-
yield* selected
|
|
260
|
+
if (stopping() || terminal !== undefined) {
|
|
261
|
+
yield* closeSlot(selected);
|
|
162
262
|
return;
|
|
163
263
|
}
|
|
164
264
|
current = selected;
|
|
@@ -167,15 +267,15 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
167
267
|
if (!publishReady(current))
|
|
168
268
|
return;
|
|
169
269
|
yield* log("Replaced lost session; local queue resumes on the new connection");
|
|
170
|
-
}).pipe(Effect.withSpan("reactor.orchestration.renewal.replace", { attributes: { "reactor.session.id": slot.source.id } }, { captureStackTrace: false }), Effect.catch(fail));
|
|
270
|
+
}).pipe(Effect.withSpan("reactor.orchestration.renewal.replace", { attributes: { "reactor.session.id": slot.source.id } }, { captureStackTrace: false }), Effect.catch((cause) => (stopping() ? Effect.void : fail(cause))));
|
|
171
271
|
const recover = (slot, cause) => Effect.gen(function* () {
|
|
172
|
-
if (slot.closed ||
|
|
272
|
+
if (slot.closed || stopping() || terminal !== undefined)
|
|
173
273
|
return;
|
|
174
274
|
if (slot === current)
|
|
175
275
|
setMedia({ _tag: "Recovering", sessionId: slot.source.id, cause });
|
|
176
276
|
yield* announce({ _tag: "Recovering", sessionId: slot.source.id, reason: cause.message });
|
|
177
277
|
yield* slot.joinCommitted(recoveryBudget(slot));
|
|
178
|
-
if (slot.closed ||
|
|
278
|
+
if (slot.closed || stopping() || terminal !== undefined)
|
|
179
279
|
return;
|
|
180
280
|
if (slot.needsReplacement()) {
|
|
181
281
|
yield* replace(slot, cause);
|
|
@@ -191,13 +291,13 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
191
291
|
return;
|
|
192
292
|
}
|
|
193
293
|
yield* slot.closeMedia;
|
|
194
|
-
if (slot.closed ||
|
|
294
|
+
if (slot.closed || stopping() || terminal !== undefined)
|
|
195
295
|
return;
|
|
196
296
|
const connected = yield* Effect.result(slot.source.reconnect.pipe(Effect.timeoutOrElse({
|
|
197
297
|
duration: recoveryBudget(slot),
|
|
198
298
|
orElse: () => Effect.fail(ReactorError.fromCode("Disconnected", "Explicit source reconnect timed out")),
|
|
199
299
|
})));
|
|
200
|
-
if (slot.closed ||
|
|
300
|
+
if (slot.closed || stopping() || terminal !== undefined)
|
|
201
301
|
return;
|
|
202
302
|
if (Result.isFailure(connected)) {
|
|
203
303
|
yield* replace(slot, connected.failure);
|
|
@@ -221,7 +321,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
221
321
|
yield* slot.replaceMedia(media, previousPressure);
|
|
222
322
|
return yield* startMedia(slot);
|
|
223
323
|
}));
|
|
224
|
-
if (slot.closed ||
|
|
324
|
+
if (slot.closed || stopping() || terminal !== undefined)
|
|
225
325
|
return;
|
|
226
326
|
if (Result.isFailure(restarted)) {
|
|
227
327
|
yield* replace(slot, restarted.failure);
|
|
@@ -240,34 +340,54 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
240
340
|
});
|
|
241
341
|
}).pipe(Effect.withSpan("reactor.orchestration.renewal.recover", { attributes: { "reactor.session.id": slot.source.id } }, { captureStackTrace: false }));
|
|
242
342
|
const scheduleRecovery = (slot, cause, mode) => Effect.gen(function* () {
|
|
243
|
-
if (slot.closed ||
|
|
343
|
+
if (slot.closed || stopping() || terminal !== undefined)
|
|
244
344
|
return;
|
|
245
345
|
if (!slot.beginRecovery(mode))
|
|
246
346
|
return;
|
|
247
|
-
yield* recover(slot, cause).pipe(commands.withPermits(1), Effect.forkIn(scope));
|
|
347
|
+
yield* recover(slot, cause).pipe(commands.withPermits(1), supervise, Effect.forkIn(scope));
|
|
248
348
|
});
|
|
249
349
|
const startMedia = (slot) => slot.startMedia({
|
|
250
350
|
video: (frame) => Effect.suspend(() => {
|
|
251
|
-
if (slot !== current ||
|
|
351
|
+
if (slot !== current || terminal !== undefined)
|
|
252
352
|
return Effect.void;
|
|
253
|
-
slot.recordVideo()
|
|
353
|
+
if (slot.recordVideo() && replacement._tag === "Ready")
|
|
354
|
+
emit({ _tag: "HandoffReady", sessionId: slot.source.id });
|
|
254
355
|
return buffer.offerVideo(frame).pipe(Effect.catch(fail));
|
|
255
356
|
}),
|
|
256
357
|
audio: (frame) => Effect.suspend(() => {
|
|
257
|
-
if (slot !== current ||
|
|
358
|
+
if (slot !== current || terminal !== undefined)
|
|
258
359
|
return Effect.void;
|
|
259
360
|
return buffer.offerAudio(frame).pipe(Effect.tap(() => Effect.sync(() => {
|
|
260
361
|
slot.recordAudio(frame.samples.length);
|
|
261
362
|
})), Effect.catch(fail));
|
|
262
363
|
}),
|
|
263
|
-
lost: (cause) => (
|
|
364
|
+
lost: (cause) => (stopping() ? Effect.void : scheduleRecovery(slot, cause, "reconnect")),
|
|
264
365
|
});
|
|
265
|
-
|
|
366
|
+
// Runs only on a fiber of its own, forked into the owner's scope.
|
|
367
|
+
const acquisition = Effect.uninterruptibleMask((restore) => Effect.gen(function* () {
|
|
368
|
+
const fiber = yield* Effect.fiber;
|
|
266
369
|
if (opened >= maxSessions)
|
|
267
370
|
return yield* ReactorError.fromCode("Overflow", "Orchestration session and cleanup-history bound reached");
|
|
371
|
+
if (stopping() || (continuous && (terminal !== undefined || !renewing)))
|
|
372
|
+
return yield* admissionClosed();
|
|
373
|
+
// Registered in the same synchronous step as the stopping check, so
|
|
374
|
+
// either close sees this attempt or this attempt sees close.
|
|
375
|
+
acquisitions.add(fiber);
|
|
376
|
+
const reservation = retention === undefined ? undefined : yield* retention.reserve;
|
|
377
|
+
const attemptIncarnation = reservation?.incarnation ?? ++incarnation;
|
|
378
|
+
const recordAttempt = (cleanup) => {
|
|
379
|
+
if (initializing && firstCleanup === undefined)
|
|
380
|
+
firstCleanup = cleanup;
|
|
381
|
+
return retention !== undefined && reservation !== undefined
|
|
382
|
+
? retention.record(reservation, cleanup)
|
|
383
|
+
: Effect.sync(() => recordCleanup(cleanup));
|
|
384
|
+
};
|
|
268
385
|
const owned = yield* Scope.make();
|
|
386
|
+
const attempt = {};
|
|
269
387
|
let acquired;
|
|
270
388
|
let acquiredSource;
|
|
389
|
+
let acquisitionCleanup;
|
|
390
|
+
let acquisitionDefect;
|
|
271
391
|
return yield* restore(Effect.gen(function* () {
|
|
272
392
|
// Replace the captured Scope explicitly in one context operation.
|
|
273
393
|
// Dependency provision order cannot move resources to the caller's owner.
|
|
@@ -276,34 +396,89 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
276
396
|
orElse: () => Effect.fail(ReactorError.fromCode("InvalidState", "Source acquisition timed out")),
|
|
277
397
|
}));
|
|
278
398
|
acquiredSource = value.source;
|
|
399
|
+
if (retention !== undefined && reservation !== undefined)
|
|
400
|
+
yield* retention.identify(reservation, value.source.id);
|
|
401
|
+
const owner = Object.freeze({
|
|
402
|
+
namespace,
|
|
403
|
+
incarnation: attemptIncarnation,
|
|
404
|
+
sessionId: value.source.id,
|
|
405
|
+
});
|
|
279
406
|
const lifetime = yield* parsed(() => duration(value.lifetime, "source lifetime", { allowInfinite: true }));
|
|
280
407
|
// A lead as long as the lifetime would prepare a replacement at once,
|
|
281
408
|
// after every open, bounded only by maxSessions.
|
|
282
409
|
if (Duration.isFinite(lifetime) && leadSeconds >= Duration.toSeconds(lifetime))
|
|
283
410
|
return yield* ReactorError.fromCode("InvalidInput", "renewal lead must be shorter than the source lifetime", { operation: "renewal", outcome: "not-submitted" });
|
|
284
|
-
if (
|
|
411
|
+
if (physicalOwners.has(value.source.id))
|
|
285
412
|
return yield* ReactorError.fromCode("InvalidInput", "Orchestration sources must have distinct session identities");
|
|
286
413
|
const media = yield* value.source.media;
|
|
287
|
-
const
|
|
414
|
+
const local = yield* SourceSlot.make({
|
|
288
415
|
source: value.source,
|
|
289
416
|
scope: owned,
|
|
290
417
|
media,
|
|
291
418
|
openedAt: monotonicMillis(clock),
|
|
292
419
|
maxSeconds: Duration.toSeconds(lifetime),
|
|
293
420
|
cleanupBudgetMs: reconnectTimeout,
|
|
294
|
-
recordCleanup,
|
|
295
|
-
|
|
421
|
+
recordCleanup: recordAttempt,
|
|
422
|
+
boundedHistory: continuous,
|
|
423
|
+
retired: (facts, retirementExit) => Effect.gen(function* () {
|
|
424
|
+
const bookkeeping = yield* Effect.exit(Effect.gen(function* () {
|
|
425
|
+
if (acquisitionCleanup !== undefined)
|
|
426
|
+
yield* recordAttempt(acquisitionCleanup);
|
|
427
|
+
if (retention === undefined || reservation === undefined)
|
|
428
|
+
return;
|
|
429
|
+
const lossExit = acquired === undefined
|
|
430
|
+
? Exit.void
|
|
431
|
+
: yield* Effect.exit(retireOwner(acquired));
|
|
432
|
+
yield* retention.finish(reservation, {
|
|
433
|
+
...facts,
|
|
434
|
+
errors: [
|
|
435
|
+
...facts.errors,
|
|
436
|
+
...(acquisitionDefect === undefined ? [] : [acquisitionDefect]),
|
|
437
|
+
...(Exit.isFailure(lossExit)
|
|
438
|
+
? [errorOf(lossExit.cause, "InvalidState", "media loss accounting")]
|
|
439
|
+
: []),
|
|
440
|
+
],
|
|
441
|
+
});
|
|
442
|
+
slots.delete(owner);
|
|
443
|
+
if (physicalOwners.get(owner.sessionId) === owner)
|
|
444
|
+
physicalOwners.delete(owner.sessionId);
|
|
445
|
+
return yield* lossExit;
|
|
446
|
+
}));
|
|
447
|
+
// The slot calls this once, however many callers wait on its close.
|
|
448
|
+
const exit = Exit.asVoidAll([retirementExit, bookkeeping]);
|
|
449
|
+
if (Exit.isFailure(exit))
|
|
450
|
+
recordFailure(attempt, exit.cause.reasons);
|
|
451
|
+
return yield* bookkeeping;
|
|
452
|
+
}),
|
|
453
|
+
retireSequences: affinity.retire(owner).pipe(Effect.asVoid),
|
|
296
454
|
});
|
|
455
|
+
const slot = Object.assign(local, { owner });
|
|
297
456
|
acquired = slot;
|
|
298
|
-
slots.set(
|
|
457
|
+
slots.set(owner, slot);
|
|
458
|
+
physicalOwners.set(owner.sessionId, owner);
|
|
299
459
|
opened++;
|
|
300
460
|
yield* slot.source.setAutoplay(false);
|
|
301
|
-
yield* slot.observe((event) => Effect.
|
|
461
|
+
yield* slot.observe((event) => Effect.gen(function* () {
|
|
302
462
|
if (event._tag === "SessionFailed")
|
|
303
|
-
return scheduleRecovery(slot, event.failure, "replace");
|
|
463
|
+
return yield* scheduleRecovery(slot, event.failure, "replace");
|
|
464
|
+
// Once the final clip has ended or arrived in full, a planned
|
|
465
|
+
// switch follows within its bounded grace, so the retiring
|
|
466
|
+
// source running dry is not starvation. Before its Ended, no
|
|
467
|
+
// switch is due yet, and that starvation is reported.
|
|
468
|
+
if (event._tag === "Starved" &&
|
|
469
|
+
slot === current &&
|
|
470
|
+
replacement._tag === "Ready" &&
|
|
471
|
+
(slot.finalClip().video !== "incomplete" ||
|
|
472
|
+
slot.finalClip().endedAgoMs !== undefined)) {
|
|
473
|
+
const next = yield* replacement.slot.source.state;
|
|
474
|
+
if (next.availability === "Ready" &&
|
|
475
|
+
next.ready.length > 0 &&
|
|
476
|
+
isIdle(yield* slot.source.state) &&
|
|
477
|
+
!(yield* openSequences(slot)))
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
304
480
|
if (event._tag !== "Starved" || slot === current)
|
|
305
481
|
emit(event);
|
|
306
|
-
return Effect.void;
|
|
307
482
|
}), (cause) => scheduleRecovery(slot, cause, "replace"));
|
|
308
483
|
yield* startMedia(slot);
|
|
309
484
|
yield* announce({
|
|
@@ -315,46 +490,127 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
315
490
|
})).pipe(Effect.onExit((exit) => Exit.isSuccess(exit)
|
|
316
491
|
? Effect.void
|
|
317
492
|
: Effect.gen(function* () {
|
|
493
|
+
// Capture an acquisition failure even if the source also
|
|
494
|
+
// returns cleanup. Conflicting reports belong to this attempt.
|
|
495
|
+
// A typed failure is the attempt's result, and cancellation
|
|
496
|
+
// is not a defect: only a Die is a failure close must report.
|
|
497
|
+
const defects = exit.cause.reasons.filter(Cause.isDieReason);
|
|
498
|
+
if (defects.length > 0) {
|
|
499
|
+
recordFailure(attempt, defects);
|
|
500
|
+
acquisitionDefect = errorOf(Cause.fromReasons(defects), "InvalidState", "source acquisition");
|
|
501
|
+
}
|
|
502
|
+
const failure = Cause.findErrorOption(exit.cause);
|
|
503
|
+
if (Option.isSome(failure) && AcquisitionFailure.is(failure.value))
|
|
504
|
+
acquisitionCleanup = { lease: failure.value.cleanup, policy: [] };
|
|
318
505
|
if (acquired !== undefined)
|
|
319
506
|
yield* closeSlot(acquired);
|
|
320
507
|
else {
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
508
|
+
const sourceExit = acquiredSource === undefined
|
|
509
|
+
? Exit.void
|
|
510
|
+
: yield* Effect.exit(acquiredSource.close.pipe(Effect.flatMap(recordAttempt)));
|
|
511
|
+
const scopeExit = yield* Effect.exit(Scope.close(owned, exit));
|
|
512
|
+
if (acquisitionCleanup !== undefined)
|
|
513
|
+
yield* recordAttempt(acquisitionCleanup);
|
|
514
|
+
if (retention !== undefined && reservation !== undefined) {
|
|
515
|
+
yield* retention.finish(reservation, {
|
|
516
|
+
accounting: "not-applicable",
|
|
517
|
+
unknownSubmissions: 0n,
|
|
518
|
+
affinity: "not-applicable",
|
|
519
|
+
scope: Exit.isSuccess(scopeExit) ? "closed" : "failed",
|
|
520
|
+
errors: [
|
|
521
|
+
...(acquisitionDefect === undefined ? [] : [acquisitionDefect]),
|
|
522
|
+
...[sourceExit, scopeExit].flatMap((result) => Exit.isFailure(result)
|
|
523
|
+
? [errorOf(result.cause, "InvalidState", "acquisition retirement")]
|
|
524
|
+
: []),
|
|
525
|
+
],
|
|
526
|
+
});
|
|
328
527
|
}
|
|
528
|
+
const retirementExit = Exit.asVoidAll([sourceExit, scopeExit]);
|
|
529
|
+
if (Exit.isFailure(retirementExit))
|
|
530
|
+
recordFailure(attempt, retirementExit.cause.reasons);
|
|
531
|
+
yield* retirementExit;
|
|
329
532
|
}
|
|
330
533
|
})));
|
|
331
|
-
})).pipe(
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
534
|
+
})).pipe(
|
|
535
|
+
// The attempt leaves the registry only after its own cleanup has run.
|
|
536
|
+
Effect.onExit(() => Effect.withFiber((fiber) => Effect.sync(() => void acquisitions.delete(fiber)))), Effect.tap((slot) => Effect.annotateCurrentSpan("reactor.session.id", slot.source.id)), Effect.withSpan("reactor.orchestration.renewal.open", {}, { captureStackTrace: false }));
|
|
537
|
+
// A forked attempt is interruptible whatever its caller is, so close and
|
|
538
|
+
// the owner's scope can stop an open that an uninterruptible caller waits on.
|
|
539
|
+
// A prepared replacement opens on its own after the tick that forks it.
|
|
540
|
+
const forkAcquisition = Effect.forkIn(acquisition, scope);
|
|
541
|
+
// To its caller, an attempt that close, stopRenewal or the owner's scope
|
|
542
|
+
// stopped is refused, not failed. Interrupted while the owner still admits
|
|
543
|
+
// sources, as by its own open, the attempt failed. A caller interrupted
|
|
544
|
+
// while it waits stops the attempt too: the handler is in place before the
|
|
545
|
+
// wait is restored, so an interruption that arrives first still reaches it.
|
|
546
|
+
const joinAcquisition = (fiber, restore) => restore(Fiber.await(fiber)).pipe(Effect.onInterrupt(() => Fiber.interrupt(fiber)), Effect.flatMap((exit) => {
|
|
547
|
+
if (Exit.isSuccess(exit))
|
|
548
|
+
return exit;
|
|
549
|
+
const failures = exit.cause.reasons.filter((reason) => !Cause.isInterruptReason(reason));
|
|
550
|
+
if (failures.length > 0)
|
|
551
|
+
return Effect.failCause(Cause.fromReasons(failures));
|
|
552
|
+
return stopping()
|
|
553
|
+
? admissionClosed()
|
|
554
|
+
: ReactorError.fromCode("InvalidState", "Source acquisition was interrupted");
|
|
555
|
+
}));
|
|
556
|
+
// The constructor and a replacement start their attempt at once, so an open
|
|
557
|
+
// that completes without waiting is published in the same turn.
|
|
558
|
+
const acquire = Effect.uninterruptibleMask((restore) => Effect.forkIn(acquisition, scope, { startImmediately: true }).pipe(Effect.flatMap((fiber) => joinAcquisition(fiber, restore))));
|
|
559
|
+
const close = yield* SourceSlot.once(Effect.uninterruptible(Effect.gen(function* () {
|
|
335
560
|
closing = true;
|
|
561
|
+
// Every registered fiber is an attempt this owner forked, never a
|
|
562
|
+
// caller's. Interrupting joins each attempt's cleanup, which records
|
|
563
|
+
// its own defects; cancelling it is not a close failure, and an
|
|
564
|
+
// Opening fiber that has not started never runs. Close does not wait
|
|
565
|
+
// for an attempt it runs on, but one issued from inside `open` cannot
|
|
566
|
+
// finish: `open` runs on a race fiber that stopping its attempt awaits.
|
|
567
|
+
const pending = new Set(acquisitions);
|
|
336
568
|
if (replacement._tag === "Opening")
|
|
337
|
-
|
|
338
|
-
yield* Effect.
|
|
569
|
+
pending.add(replacement.fiber);
|
|
570
|
+
yield* Effect.withFiber((self) => {
|
|
571
|
+
pending.delete(self);
|
|
572
|
+
return Fiber.interruptAll(pending);
|
|
573
|
+
});
|
|
574
|
+
for (const slot of [...slots.values()])
|
|
575
|
+
yield* Effect.exit(closeSlot(slot));
|
|
576
|
+
const exits = [...attemptFailures.values()].map((reasons) => Exit.failCause(Cause.fromReasons(reasons)));
|
|
339
577
|
buffer.end();
|
|
340
578
|
setMedia({ _tag: "Closed" });
|
|
341
579
|
observations.end();
|
|
342
|
-
// The reader delivers what was announced before close, within a bound.
|
|
343
580
|
if (renewalReader !== undefined)
|
|
344
581
|
yield* Fiber.await(renewalReader).pipe(Effect.interruptible, Effect.timeout("1 second"), Effect.ignore);
|
|
345
|
-
|
|
346
|
-
|
|
582
|
+
if (retention === undefined)
|
|
583
|
+
finalReport = Object.freeze({ sessions: Object.freeze([...cleanups]) });
|
|
584
|
+
else
|
|
585
|
+
finalSummary = yield* retention.conclude;
|
|
586
|
+
// The closed owner stays current, so final loss totals remain readable.
|
|
587
|
+
replacement = { _tag: "Absent" };
|
|
588
|
+
return yield* Exit.asVoidAll(exits);
|
|
347
589
|
})));
|
|
348
590
|
yield* Effect.addFinalizer(() => close);
|
|
349
|
-
current = yield* acquire.pipe(Effect.tap((slot) => slot.source.setAutoplay(true)),
|
|
591
|
+
current = yield* acquire.pipe(Effect.tap((slot) => slot.source.setAutoplay(true)),
|
|
592
|
+
// Fenced as publishReady is: construction returns no handle whose owner
|
|
593
|
+
// stopped while its first source was activated, and close retires it.
|
|
594
|
+
Effect.tap(() => Effect.gen(function* () {
|
|
595
|
+
if (stopping())
|
|
596
|
+
return yield* admissionClosed();
|
|
597
|
+
})), Effect.catch((cause) => Effect.exit(close).pipe(Effect.flatMap((closed) => {
|
|
350
598
|
// An AcquisitionFailure from `open` already carries the lease it
|
|
351
599
|
// recorded, by reference; any later failure takes the recorded lease.
|
|
352
|
-
const lease =
|
|
353
|
-
|
|
600
|
+
const lease = firstCleanup?.lease;
|
|
601
|
+
const failure = AcquisitionFailure.is(cause) || (lease === undefined && ReactorError.is(cause))
|
|
354
602
|
? cause
|
|
355
|
-
: AcquisitionFailure.from(cause, lease ?? noAcquisition)
|
|
603
|
+
: AcquisitionFailure.from(cause, lease ?? noAcquisition);
|
|
604
|
+
// A defect of the close that retired the attempt stays beside the
|
|
605
|
+
// typed failure, which the caller still handles.
|
|
606
|
+
return Exit.isSuccess(closed)
|
|
607
|
+
? Effect.fail(failure)
|
|
608
|
+
: Effect.failCause(Cause.combine(Cause.fail(failure), closed.cause));
|
|
356
609
|
}))));
|
|
610
|
+
initializing = false;
|
|
611
|
+
firstCleanup = undefined;
|
|
357
612
|
// The first owner's loss from its start is the output's: no baseline.
|
|
613
|
+
outputOwner = current.owner;
|
|
358
614
|
setMedia({
|
|
359
615
|
_tag: "Ready",
|
|
360
616
|
sessionId: current.source.id,
|
|
@@ -372,7 +628,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
372
628
|
});
|
|
373
629
|
const route = (request) => Effect.gen(function* () {
|
|
374
630
|
const values = yield* Effect.suspend(() => Effect.forEach([...slots.values()], (slot) => Effect.map(slot.source.state, (state) => ({
|
|
375
|
-
owner: slot.
|
|
631
|
+
owner: slot.owner,
|
|
376
632
|
state,
|
|
377
633
|
accepted: slot.accepted,
|
|
378
634
|
closed: slot.closed,
|
|
@@ -385,25 +641,43 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
385
641
|
: current;
|
|
386
642
|
if (preferred === undefined)
|
|
387
643
|
return yield* PolicyFailure.refuse("SessionRecovering", "No source is ready");
|
|
388
|
-
return yield* resolve(request, values, preferred.
|
|
644
|
+
return yield* resolve(request, values, preferred.owner, binding);
|
|
389
645
|
});
|
|
390
646
|
const sequenceError = (error) => PolicyFailure.sequence(error.sequenceId, error.code);
|
|
391
|
-
const prepare = (input) => Effect.gen(function* () {
|
|
647
|
+
const prepare = (input, expectedSessionId) => Effect.gen(function* () {
|
|
648
|
+
const expectedOwner = expectedSessionId === undefined ? undefined : physicalOwners.get(expectedSessionId);
|
|
392
649
|
const request = yield* captureRequest(input);
|
|
393
650
|
const id = `${namespace}:${++submissionSequence}`;
|
|
394
651
|
const gate = yield* Semaphore.make(1);
|
|
652
|
+
const logical = logicalSubmission(id);
|
|
653
|
+
let selectedOwner;
|
|
395
654
|
let active;
|
|
396
655
|
let activeOwner;
|
|
397
656
|
const submit = gate.withPermit(Effect.gen(function* () {
|
|
657
|
+
const completed = logical.outcome();
|
|
658
|
+
if (completed !== undefined)
|
|
659
|
+
return yield* completed;
|
|
398
660
|
if (active === undefined) {
|
|
399
661
|
// A committed submission remains a readable outcome after the handle
|
|
400
662
|
// closes. Only selecting or committing fresh work needs a live owner.
|
|
401
663
|
yield* guard("enqueue", Effect.void);
|
|
402
664
|
const decision = yield* route(request).pipe(commands.withPermits(1));
|
|
665
|
+
if (expectedSessionId !== undefined && decision.owner !== expectedOwner)
|
|
666
|
+
return yield* PolicyFailure.refuse("RouteChanged", "The selected source changed before dispatch");
|
|
667
|
+
if (continuous && selectedOwner !== undefined && selectedOwner !== decision.owner) {
|
|
668
|
+
// The selection stays fenced to its incarnation. While that
|
|
669
|
+
// source is live, only the route moved; it has not retired.
|
|
670
|
+
const selected = slots.get(selectedOwner);
|
|
671
|
+
return yield* selected !== undefined && !selected.closed
|
|
672
|
+
? PolicyFailure.refuse("RouteChanged", "The selected source changed before dispatch")
|
|
673
|
+
: PolicyFailure.refuse("SessionRetired", "Selected source incarnation was retired");
|
|
674
|
+
}
|
|
675
|
+
selectedOwner = decision.owner;
|
|
403
676
|
const target = slots.get(decision.owner);
|
|
404
677
|
if (target === undefined)
|
|
405
678
|
return yield* PolicyFailure.refuse("SessionRetired", "Selected source was retired");
|
|
406
679
|
const sequence = request.sequence;
|
|
680
|
+
let resultAccounted = false;
|
|
407
681
|
active = yield* target.source.prepareRouted({ request, position: decision.position }, {
|
|
408
682
|
commit: (submissionId) => commands.withPermit(guard("enqueue", Effect.gen(function* () {
|
|
409
683
|
const checked = yield* route(request);
|
|
@@ -415,28 +689,43 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
415
689
|
return yield* Effect.die(new Error("Source committed before returning its inert submission"));
|
|
416
690
|
yield* target.register(active, Effect.gen(function* () {
|
|
417
691
|
if (sequence !== undefined) {
|
|
418
|
-
yield* affinity.bind(sequence.id, target.
|
|
692
|
+
yield* affinity.bind(sequence.id, target.owner);
|
|
419
693
|
yield* affinity.begin(sequence.id, sequence.memberId ?? submissionId);
|
|
420
694
|
}
|
|
421
|
-
}).pipe(Effect.mapError(sequenceError)))
|
|
695
|
+
}).pipe(Effect.mapError(sequenceError)), (exit) => {
|
|
696
|
+
logical.complete(exit);
|
|
697
|
+
active = undefined;
|
|
698
|
+
activeOwner = undefined;
|
|
699
|
+
});
|
|
422
700
|
}))),
|
|
423
701
|
result: (submissionId, result) => Effect.gen(function* () {
|
|
424
|
-
|
|
702
|
+
if (resultAccounted)
|
|
703
|
+
return;
|
|
704
|
+
resultAccounted = true;
|
|
705
|
+
// An accepted identity past the continuous bound is a
|
|
706
|
+
// source-contract failure, as for Started: it is never
|
|
707
|
+
// retained, its member cannot be attributed and the source
|
|
708
|
+
// is replaced.
|
|
709
|
+
const recorded = yield* Effect.result(target.recordResult(result));
|
|
425
710
|
if (sequence !== undefined) {
|
|
426
711
|
const memberId = sequence.memberId ?? submissionId;
|
|
427
|
-
const account = Result.
|
|
428
|
-
? affinity.
|
|
429
|
-
: result
|
|
430
|
-
? affinity.
|
|
431
|
-
:
|
|
712
|
+
const account = Result.isFailure(recorded)
|
|
713
|
+
? affinity.uncertain(sequence.id, memberId, recorded.failure.message)
|
|
714
|
+
: Result.isSuccess(result)
|
|
715
|
+
? affinity.accepted(sequence.id, memberId, result.success, sequence.final)
|
|
716
|
+
: result.failure.context.outcome === "unknown"
|
|
717
|
+
? affinity.uncertain(sequence.id, memberId, result.failure.message)
|
|
718
|
+
: affinity.rejected(sequence.id, memberId, result.failure.message, sequence.final);
|
|
432
719
|
yield* account.pipe(Effect.catch((cause) => Effect.gen(function* () {
|
|
433
720
|
target.markIndeterminate();
|
|
434
|
-
yield* affinity.retire(target.
|
|
721
|
+
yield* affinity.retire(target.owner);
|
|
435
722
|
yield* fail(ReactorError.fromCode("InvalidState", "Committed sequence accounting failed", { detail: cause }));
|
|
436
723
|
})));
|
|
437
724
|
}
|
|
438
725
|
target.finishAccounting();
|
|
439
|
-
if (Result.isFailure(
|
|
726
|
+
if (Result.isFailure(recorded))
|
|
727
|
+
yield* scheduleRecovery(target, recorded.failure, "replace");
|
|
728
|
+
else if (Result.isFailure(result) &&
|
|
440
729
|
result.failure.context.outcome === "unknown") {
|
|
441
730
|
yield* scheduleRecovery(target, result.failure, "replace");
|
|
442
731
|
}
|
|
@@ -446,29 +735,55 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
446
735
|
}
|
|
447
736
|
const selected = active;
|
|
448
737
|
return yield* selected.submit.pipe(Effect.onExit(() => selected.state.pipe(Effect.flatMap((state) => Effect.sync(() => {
|
|
449
|
-
if (state._tag === "Completed")
|
|
738
|
+
if (state._tag === "Completed") {
|
|
739
|
+
logical.complete(state.exit);
|
|
450
740
|
activeOwner?.forgetCompleted(selected);
|
|
741
|
+
active = undefined;
|
|
742
|
+
activeOwner = undefined;
|
|
743
|
+
}
|
|
451
744
|
if (state._tag === "Prepared" && active === selected) {
|
|
452
745
|
active = undefined;
|
|
453
746
|
activeOwner = undefined;
|
|
454
747
|
}
|
|
455
748
|
})))));
|
|
456
749
|
}));
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
? Effect.succeed({ _tag: "Prepared" })
|
|
462
|
-
: active.state),
|
|
463
|
-
};
|
|
750
|
+
logical.attach(submit, Effect.suspend(() => active === undefined
|
|
751
|
+
? Effect.succeed({ _tag: "Prepared" })
|
|
752
|
+
: active.state));
|
|
753
|
+
return logical.submission;
|
|
464
754
|
});
|
|
465
755
|
const state = Effect.gen(function* () {
|
|
756
|
+
if (!closing && terminal !== undefined && Exit.isFailure(terminal))
|
|
757
|
+
return yield* Effect.failCause(terminal.cause);
|
|
466
758
|
const live = [...slots.values()].filter((slot) => !slot.closed);
|
|
467
|
-
const values = yield* Effect.forEach(live, (slot) => slot.source.state);
|
|
759
|
+
const values = (yield* Effect.forEach(live, (slot) => slot.source.state)).map((value, index) => withSessionId(value, live[index].source.id));
|
|
468
760
|
const primary = values[live.indexOf(current)] ?? emptyState();
|
|
761
|
+
const next = replacement._tag === "Ready" ? replacement.slot : undefined;
|
|
762
|
+
const preferred = next !== undefined && current !== undefined && !(yield* openSequences(current))
|
|
763
|
+
? next
|
|
764
|
+
: current;
|
|
469
765
|
const builds = values.flatMap((value) => Option.isSome(value.building) ? [value.building.value] : []);
|
|
470
766
|
return Object.freeze({
|
|
471
767
|
...primary,
|
|
768
|
+
sessions: Object.freeze(values.map((value, index) => ({
|
|
769
|
+
sessionId: live[index].source.id,
|
|
770
|
+
availability: value.availability,
|
|
771
|
+
}))),
|
|
772
|
+
preferredSessionId: Option.fromUndefinedOr(preferred?.source.id),
|
|
773
|
+
retiringSessionId: next !== undefined && current !== undefined && !current.closed
|
|
774
|
+
? Option.some(current.source.id)
|
|
775
|
+
: Option.none(),
|
|
776
|
+
// No media condition: a short final clip holds the switch only for the
|
|
777
|
+
// grace past its Ended, or past the source first reported idle. Gating
|
|
778
|
+
// on the playing clip's frames instead could hold only in the instant
|
|
779
|
+
// between its last frame and Ended, so a lossy clip let the retiring
|
|
780
|
+
// source roll into its next filler.
|
|
781
|
+
handoffReady: next !== undefined &&
|
|
782
|
+
preferred === next &&
|
|
783
|
+
current !== undefined &&
|
|
784
|
+
!current.closed &&
|
|
785
|
+
!current.recovering &&
|
|
786
|
+
primary.availability === "Ready",
|
|
472
787
|
availability: values.some((value) => value.availability === "Unavailable")
|
|
473
788
|
? "Unavailable"
|
|
474
789
|
: values.some((value) => value.availability === "Synchronizing")
|
|
@@ -501,6 +816,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
501
816
|
const engine = {
|
|
502
817
|
prepare,
|
|
503
818
|
enqueue: (request) => prepare(request).pipe(Effect.flatMap((submission) => submission.submit)),
|
|
819
|
+
enqueueOnSource: (request, expectedSessionId) => prepare(request, expectedSessionId).pipe(Effect.flatMap((submission) => submission.submit)),
|
|
504
820
|
state,
|
|
505
821
|
events: Stream.filterMap(observations.stream(), engineOnly),
|
|
506
822
|
observe: (options) => observations.observeWith(state, options).pipe(Effect.map(({ initial, events }) => ({
|
|
@@ -508,6 +824,22 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
508
824
|
events: Stream.filterMap(events, engineOnly),
|
|
509
825
|
}))),
|
|
510
826
|
failure: Deferred.await(fatal),
|
|
827
|
+
stopRenewal: Effect.gen(function* () {
|
|
828
|
+
// Fence allocation before waiting on a command already in progress.
|
|
829
|
+
renewing = false;
|
|
830
|
+
yield* commands.withPermit(Effect.gen(function* () {
|
|
831
|
+
if (replacement._tag !== "Opening")
|
|
832
|
+
return;
|
|
833
|
+
const pending = replacement.fiber;
|
|
834
|
+
// It stays Opening until joined, so a concurrent close joins it too.
|
|
835
|
+
yield* Fiber.interrupt(pending);
|
|
836
|
+
const result = yield* Fiber.await(pending);
|
|
837
|
+
if (replacement._tag === "Opening" && replacement.fiber === pending)
|
|
838
|
+
replacement = { _tag: "Absent" };
|
|
839
|
+
if (Exit.isSuccess(result))
|
|
840
|
+
yield* closeSlot(result.value);
|
|
841
|
+
}));
|
|
842
|
+
}),
|
|
511
843
|
setAutoplay: (enabled) => commands.withPermit(guard("set_autoplay", Effect.gen(function* () {
|
|
512
844
|
autoplay = enabled;
|
|
513
845
|
if (current !== undefined && !current.closed)
|
|
@@ -538,7 +870,9 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
538
870
|
const value = yield* slot.source.state;
|
|
539
871
|
preceding += queue === "generation" ? generation(value).length : value.ready.length;
|
|
540
872
|
}
|
|
541
|
-
|
|
873
|
+
if (position < preceding || position >= preceding + own.length)
|
|
874
|
+
return yield* PolicyFailure.refuse("InvalidRequest", "Move position is outside the clip's session range", "move");
|
|
875
|
+
yield* selected.slot.source.move(id, position - preceding);
|
|
542
876
|
}))),
|
|
543
877
|
setCanvas: (canvas) => commands.withPermit(guard("set_canvas", Effect.gen(function* () {
|
|
544
878
|
if (!isIdle(yield* state))
|
|
@@ -550,7 +884,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
550
884
|
};
|
|
551
885
|
const tick = commands
|
|
552
886
|
.withPermit(Effect.gen(function* () {
|
|
553
|
-
if (
|
|
887
|
+
if (stopping() ||
|
|
554
888
|
current === undefined ||
|
|
555
889
|
current.closed ||
|
|
556
890
|
current.recovering ||
|
|
@@ -560,6 +894,10 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
560
894
|
const result = replacement.fiber.pollUnsafe();
|
|
561
895
|
if (result !== undefined) {
|
|
562
896
|
replacement = { _tag: "Absent" };
|
|
897
|
+
// Close or the owner's scope stopped this attempt, and close
|
|
898
|
+
// retires whatever it opened: that is no failed setup.
|
|
899
|
+
if (stopping())
|
|
900
|
+
return;
|
|
563
901
|
if (Exit.isSuccess(result)) {
|
|
564
902
|
replacement = { _tag: "Ready", slot: result.value };
|
|
565
903
|
openFailures = 0;
|
|
@@ -567,6 +905,13 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
567
905
|
yield* announce({ _tag: "Prepared" });
|
|
568
906
|
}
|
|
569
907
|
else {
|
|
908
|
+
if (continuous) {
|
|
909
|
+
const error = Cause.findErrorOption(result.cause);
|
|
910
|
+
if (Option.isSome(error) &&
|
|
911
|
+
ReactorError.is(error.value) &&
|
|
912
|
+
error.value.reason._tag === "Overflow")
|
|
913
|
+
return yield* fail(error.value);
|
|
914
|
+
}
|
|
570
915
|
openFailures++;
|
|
571
916
|
retryAt = monotonicMillis(clock) + 5000;
|
|
572
917
|
yield* announce({
|
|
@@ -580,7 +925,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
580
925
|
}
|
|
581
926
|
}
|
|
582
927
|
const decision = decideRenewal({
|
|
583
|
-
running: !
|
|
928
|
+
running: !stopping() && terminal === undefined,
|
|
584
929
|
now: monotonicMillis(clock),
|
|
585
930
|
current,
|
|
586
931
|
replacement: replacement._tag === "Ready" ? replacement.slot.phase : replacement._tag,
|
|
@@ -596,7 +941,8 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
596
941
|
}));
|
|
597
942
|
return;
|
|
598
943
|
case "Prepare":
|
|
599
|
-
|
|
944
|
+
if (renewing)
|
|
945
|
+
replacement = { _tag: "Opening", fiber: yield* forkAcquisition };
|
|
600
946
|
return;
|
|
601
947
|
case "InspectHandoff":
|
|
602
948
|
break;
|
|
@@ -610,16 +956,21 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
610
956
|
nextState.availability !== "Ready" ||
|
|
611
957
|
nextState.ready.length === 0)
|
|
612
958
|
return;
|
|
613
|
-
|
|
614
|
-
|
|
959
|
+
// The provider reports nothing playing: a lost Ended must not hold
|
|
960
|
+
// the switch until expiry, so the grace also starts here.
|
|
961
|
+
current.observedIdle();
|
|
962
|
+
const evidence = handoffEvidence({
|
|
615
963
|
sequenceOpen: false,
|
|
616
964
|
currentIdle: isIdle(oldState),
|
|
617
965
|
replacementReady: nextState.availability === "Ready" && nextState.ready.length > 0,
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
966
|
+
finalClip: current.finalClip(),
|
|
967
|
+
graceMs: handoffGraceMs,
|
|
968
|
+
});
|
|
969
|
+
if (evidence === undefined)
|
|
622
970
|
return;
|
|
971
|
+
const handoff = Object.freeze({ ...evidence, replacementSessionId: next.source.id });
|
|
972
|
+
// Read only for a switch that happens: it can wait on source pressure.
|
|
973
|
+
const tail = yield* retired(current);
|
|
623
974
|
const old = current;
|
|
624
975
|
// The switch is traced; the tick that found it is not.
|
|
625
976
|
yield* Effect.gen(function* () {
|
|
@@ -632,7 +983,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
632
983
|
yield* closeSlot(old);
|
|
633
984
|
if (!activated)
|
|
634
985
|
return;
|
|
635
|
-
yield* announce({ _tag: "Switched", ...tail });
|
|
986
|
+
yield* announce({ _tag: "Switched", ...tail, handoff });
|
|
636
987
|
yield* log("Switched prepared sessions at a sequence boundary");
|
|
637
988
|
}).pipe(Effect.withSpan("reactor.orchestration.renewal.switch", {
|
|
638
989
|
attributes: {
|
|
@@ -644,14 +995,18 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
644
995
|
.pipe(Effect.catch(fail));
|
|
645
996
|
// Spaced polling, not a fixed rate: each tick runs 100 ms after the previous one
|
|
646
997
|
// finished, so a slow tick delays the next instead of overlapping it.
|
|
647
|
-
yield* Effect.forever(Effect.sleep(100).pipe(Effect.andThen(tick))).pipe(Effect.forkIn(scope));
|
|
998
|
+
yield* Effect.forever(Effect.sleep(100).pipe(Effect.andThen(tick))).pipe(supervise, Effect.forkIn(scope));
|
|
648
999
|
const pressure = Effect.suspend(() => {
|
|
649
1000
|
if (current === undefined)
|
|
650
1001
|
return Effect.fail(ReactorError.fromCode("InvalidState", "No active media source"));
|
|
651
1002
|
const owner = current;
|
|
652
1003
|
return owner.pressure.pipe(Effect.flatMap((source) => {
|
|
653
1004
|
const queued = buffer.pressure();
|
|
654
|
-
|
|
1005
|
+
// A retired owner's loss is already in the output's totals, as the
|
|
1006
|
+
// closed continuous owner's is once its retirement completes.
|
|
1007
|
+
const loss = outputOwner === owner.owner
|
|
1008
|
+
? SourceSlot.addLoss(outputLoss, SourceSlot.subtractLoss(SourceSlot.lossOf(Result.succeed(source)), ownerBaseline))
|
|
1009
|
+
: outputLoss;
|
|
655
1010
|
if (loss.video === null || loss.audio === null || loss.readers === null)
|
|
656
1011
|
return Effect.fail(ReactorError.fromCode("InvalidState", "A former media owner's loss totals are unknown"));
|
|
657
1012
|
return Effect.succeed({
|
|
@@ -666,7 +1021,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
666
1021
|
});
|
|
667
1022
|
}));
|
|
668
1023
|
});
|
|
669
|
-
|
|
1024
|
+
const facets = {
|
|
670
1025
|
engine,
|
|
671
1026
|
media: {
|
|
672
1027
|
video: buffer.video,
|
|
@@ -674,20 +1029,52 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
674
1029
|
pressure,
|
|
675
1030
|
videoFramesPerSecond: current.media.videoFramesPerSecond,
|
|
676
1031
|
},
|
|
677
|
-
mediaState: Effect.
|
|
1032
|
+
mediaState: Effect.suspend(() => !closing && terminal !== undefined && Exit.isFailure(terminal)
|
|
1033
|
+
? Effect.failCause(terminal.cause)
|
|
1034
|
+
: Effect.succeed(mediaState)),
|
|
678
1035
|
observe: (options) => observations.observeWith(Effect.map(state, (engine) => ({ engine, media: mediaState })), options),
|
|
679
1036
|
sessionId: Effect.sync(() => current === undefined || current.closed ? Option.none() : Option.some(current.source.id)),
|
|
680
|
-
close,
|
|
681
|
-
cleanup: Effect.sync(() => finalReport === undefined ? Option.none() : Option.some(finalReport)),
|
|
682
1037
|
sequences: {
|
|
683
|
-
get: affinity
|
|
684
|
-
|
|
1038
|
+
get: (id) => affinity
|
|
1039
|
+
.get(id)
|
|
1040
|
+
.pipe(Effect.map((entry) => entry === undefined
|
|
1041
|
+
? undefined
|
|
1042
|
+
: Object.freeze({ ...entry, owner: entry.owner.sessionId }))),
|
|
1043
|
+
snapshots: affinity.snapshots.pipe(Effect.map((entries) => Object.freeze(entries.map((entry) => Object.freeze({ ...entry, owner: entry.owner.sessionId }))))),
|
|
685
1044
|
seal: affinity.seal,
|
|
686
1045
|
acknowledgeIndeterminate: affinity.acknowledgeIndeterminate,
|
|
687
1046
|
release: affinity.release,
|
|
688
1047
|
},
|
|
689
1048
|
};
|
|
1049
|
+
return {
|
|
1050
|
+
legacy: {
|
|
1051
|
+
...facets,
|
|
1052
|
+
close: close.pipe(Effect.andThen(Effect.sync(() => {
|
|
1053
|
+
if (finalReport === undefined)
|
|
1054
|
+
throw new Error("Missing legacy cleanup report");
|
|
1055
|
+
return finalReport;
|
|
1056
|
+
}))),
|
|
1057
|
+
cleanup: Effect.sync(() => Option.fromUndefinedOr(finalReport)),
|
|
1058
|
+
},
|
|
1059
|
+
continuous: {
|
|
1060
|
+
...facets,
|
|
1061
|
+
close: close.pipe(Effect.andThen(Effect.sync(() => {
|
|
1062
|
+
if (finalSummary === undefined)
|
|
1063
|
+
throw new Error("Missing continuous cleanup summary");
|
|
1064
|
+
return finalSummary;
|
|
1065
|
+
}))),
|
|
1066
|
+
cleanup: Effect.sync(() => Option.fromUndefinedOr(finalSummary)),
|
|
1067
|
+
},
|
|
1068
|
+
};
|
|
690
1069
|
});
|
|
1070
|
+
/** Legacy renewal retains complete cleanup history and defaults to 64 successful opens. */
|
|
1071
|
+
export const make = (options) => makeOwner(options, false).pipe(Effect.map((owner) => owner.legacy));
|
|
1072
|
+
/**
|
|
1073
|
+
* Bounded continuous renewal. Adapters must meet ContinuousOptions' unique-ID
|
|
1074
|
+
* precondition. Retired anchors no longer retained can refuse as missing rather
|
|
1075
|
+
* than SessionRetired; captured operations remain fenced to their exact incarnation.
|
|
1076
|
+
*/
|
|
1077
|
+
export const makeContinuous = (options) => makeOwner(options, true).pipe(Effect.map((owner) => owner.continuous));
|
|
691
1078
|
/**
|
|
692
1079
|
* The handle's `Engine`, `Media` and `Handle` services from one orchestration,
|
|
693
1080
|
* so the three can never come from two paid session chains. Each build of the
|