reactor-effect-client 0.5.0 → 0.6.0

package/dist/orchestration/scheduler.js

@@ -10,8 +10,10 @@ import * as PubSub from "effect/PubSub";
 import * as Queue from "effect/Queue";
 import * as Result from "effect/Result";
 import * as Schema from "effect/Schema";
+import * as Scope from "effect/Scope";
 import * as Stream from "effect/Stream";
 import * as SubscriptionRef from "effect/SubscriptionRef";
+import { take as takeQueue } from "../_internal/queue.js";
 import { duration } from "../duration.js";
 import { parsedInput, ReactorError } from "../errors.js";
 import { monotonicMillis } from "./elapsed.js";
@@ -51,9 +53,8 @@ const ownedData = (input, fields, name) => {
         return Result.fail(invalid(`${name} has unsupported fields or accessors`));
     return Result.succeed(Object.fromEntries(Object.entries(descriptors).map(([field, descriptor]) => [field, descriptor.value])));
 };
-const captureDuration = (input) => Effect.gen(function* () {
-    if (input === null || typeof input !== "object" || Duration.isDuration(input))
-        return input;
+/** A Duration tuple or object read from own data properties; no accessor runs. */
+const durationData = (input) => {
     if (Array.isArray(input)) {
         const descriptors = Object.getOwnPropertyDescriptors(input);
         if (Object.getOwnPropertySymbols(input).length > 0 ||
@@ -61,20 +62,44 @@ const captureDuration = (input) => Effect.gen(function* () {
             descriptors.length?.value !== 2 ||
             descriptors["0"] === undefined ||
             descriptors["1"] === undefined)
-            return yield* invalid("Duration tuple has unsupported fields or accessors");
-        return [descriptors["0"].value, descriptors["1"].value];
+            return Result.fail(invalid("Duration tuple has unsupported fields or accessors"));
+        const seconds = descriptors["0"].value;
+        const nanos = descriptors["1"].value;
+        return Result.succeed([seconds, nanos]);
     }
-    return (yield* Effect.fromResult(ownedData(input, [
-        "weeks",
-        "days",
-        "hours",
-        "minutes",
-        "seconds",
-        "milliseconds",
-        "microseconds",
-        "nanoseconds",
-    ], "Duration")));
+    const fields = ownedData(input, ["weeks", "days", "hours", "minutes", "seconds", "milliseconds", "microseconds", "nanoseconds"], "Duration");
+    return Result.isFailure(fields) ? Result.fail(fields.failure) : Result.succeed(fields.success);
+};
+const captureDuration = (input) => Effect.gen(function* () {
+    if (input === null || typeof input !== "object" || Duration.isDuration(input))
+        return input;
+    return yield* Effect.fromResult(durationData(input));
 });
+/**
+ * A Duration option read as caller data: a data property, own or inherited,
+ * captured without running any accessor. Rejections throw for `parsedInput`.
+ */
+const optionDuration = (options, field, name) => {
+    let descriptor;
+    for (let owner = options; descriptor === undefined && owner !== null; owner = Object.getPrototypeOf(owner))
+        descriptor = Object.getOwnPropertyDescriptor(owner, field);
+    if (descriptor === undefined)
+        return undefined;
+    const reject = () => ReactorError.fromCode("InvalidInput", `${name} must be a Duration of data properties`);
+    if (!("value" in descriptor))
+        throw reject();
+    const value = descriptor.value;
+    if (value === undefined)
+        return undefined;
+    if (value === null)
+        throw reject();
+    if (typeof value !== "object" || Duration.isDuration(value))
+        return value;
+    const captured = durationData(value);
+    if (Result.isFailure(captured))
+        throw reject();
+    return captured.success;
+};
 const requestDuration = (input, name, allowZero = false) => Effect.gen(function* () {
     const captured = yield* captureDuration(input);
     return yield* parsedInput(() => Duration.toMillis(duration(captured, name, { allowZero, allowNegative: name === "startBy" })), "submit").pipe(Effect.mapError((error) => invalid(error.message)));
@@ -178,11 +203,13 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             throw ReactorError.fromCode("InvalidInput", "Scheduler lanes must be unique and nonempty");
         return options.lanes.map((lane) => lane.name);
     }, "makeScheduler");
-    const { floorSeconds, targetSeconds, maxBuildsInFlight, maxHistory } = yield* parsedInput(() => {
+    const { floorSeconds, targetSeconds, maxBuildsInFlight, maxHistory, unknownRecoveryMs } = yield* parsedInput(() => {
         const floorSeconds = Duration.toMillis(duration(options.filler.runway.floor, "runway floor", { allowZero: true })) / 1000;
         const targetSeconds = Duration.toMillis(duration(options.filler.runway.target, "runway target")) / 1000;
         const maxBuildsInFlight = options.maxBuildsInFlight ?? 1;
         const maxHistory = options.maxHistory ?? 4096;
+        const unknownRecoveryMs = Duration.toMillis(duration(optionDuration(options, "unknownRecoveryTimeout", "Unknown recovery timeout") ??
+            "60 seconds", "unknown recovery timeout", { maximum: "10 minutes" }));
         if (floorSeconds > targetSeconds ||
             !Number.isSafeInteger(maxBuildsInFlight) ||
             maxBuildsInFlight < 1 ||
@@ -191,7 +218,7 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             maxHistory < 0 ||
             maxHistory > 65_536)
             throw ReactorError.fromCode("InvalidInput", "Scheduler runway or build cap is invalid");
-        return { floorSeconds, targetSeconds, maxBuildsInFlight, maxHistory };
+        return { floorSeconds, targetSeconds, maxBuildsInFlight, maxHistory, unknownRecoveryMs };
     }, "makeScheduler");
     const captureFiller = (index, runway, targetSeconds) => Effect.gen(function* () {
         const request = yield* captureRequest(options.filler.clip({
@@ -209,7 +236,8 @@ export const makeScheduler = (options) => Effect.gen(function* () {
     });
     let fillerIndex = 0;
     let fillerRetryAtMs = 0;
-    let unknownFiller;
+    const unknownFillers = new Map();
+    const maxUnknownFillers = 4096;
     let upcomingFiller = yield* captureFiller(0, 0);
     const inbox = yield* Queue.unbounded();
     const commandQueue = yield* Queue.unbounded();
@@ -229,6 +257,7 @@ export const makeScheduler = (options) => Effect.gen(function* () {
     });
     const initialized = yield* Deferred.make();
     const stopped = yield* Deferred.make();
+    const settled = yield* Deferred.make();
     const closedCall = Deferred.await(stopped).pipe(Effect.flatMap((cause) => PolicyFailure.refuse("SessionClosed", cause.message)));
     const items = new Map();
     const history = new Map();
@@ -248,6 +277,9 @@ export const makeScheduler = (options) => Effect.gen(function* () {
     const drainReplies = [];
     const pendingWithdrawals = new Map();
     const pendingBuilds = new Set();
+    // A Build whose enqueue has started and whose result the worker has not
+    // yet seen. The provider may still accept it after the scheduler ends.
+    let dispatching;
     const pendingAtDeferrals = new Set();
     const pendingItemRemovals = new Set();
     const pendingFillerRemovals = new Set();
@@ -262,7 +294,9 @@ export const makeScheduler = (options) => Effect.gen(function* () {
     let drained = false;
     let observationReady = false;
     let blockedMove;
-    const emit = (entry, status) => Effect.gen(function* () {
+    const emit = (entry, status, terminal = false) => Effect.gen(function* () {
+        if (ended !== undefined && !terminal)
+            return;
         if (JSON.stringify(entry.status) === JSON.stringify(status))
             return;
         entry.status = status;
@@ -275,7 +309,10 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                 : status._tag === "Unknown" && status.terminal !== true
                     ? "Unknown"
                     : "Terminal";
-        yield* PubSub.publish(events, {
+        // Publish in the step that records it (the PubSub is unbounded): a claim
+        // from another fiber cannot land between the two, so asRun never carries
+        // a settlement ahead of a recorded status, nor omits one.
+        PubSub.publishUnsafe(events, {
             key: entry.key,
             at: clock.currentTimeMillisUnsafe(),
             status,
@@ -293,31 +330,165 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             yield* Deferred.succeed(entry.outcomeWaiter, status);
         }
     });
-    const closeActor = (cause) => Effect.gen(function* () {
+    // The claim is synchronous; interruption masking alone would still allow
+    // two closers to publish conflicting results. Join only local bookkeeping,
+    // never the worker scope (a worker can itself own terminal settlement).
+    const terminate = (terminal) => Effect.suspend(() => {
         if (ended !== undefined)
-            return;
-        ended = cause;
+            return Deferred.await(settled);
+        ended = terminal;
         accepting = false;
-        yield* Deferred.succeed(stopped, cause);
-        for (const item of items.values()) {
-            if (item.phase === "Terminal")
-                continue;
-            if (item.phase === "Unknown") {
-                yield* emit(item, { _tag: "Unknown", terminal: true });
-                item.phase = "Terminal";
+        // Wake the watchdog so an episode left armed by another owner cannot
+        // keep re-arming its timer.
+        Deferred.doneUnsafe(recoveryChanged, Exit.void);
+        return Effect.gen(function* () {
+            for (const item of items.values()) {
+                if (Exit.isFailure(terminal)) {
+                    yield* Deferred.failCause(item.startedWaiter, terminal.cause);
+                    yield* Deferred.failCause(item.outcomeWaiter, terminal.cause);
+                    yield* Deferred.failCause(item.firstDecisiveWaiter, terminal.cause);
+                    continue;
+                }
+                // emit records and publishes a status before completing its waits,
+                // and a claim can land in between: a completion can resume an
+                // observer that closes the owner, and another fiber can claim while
+                // the actor is preempted. Finish those waits from the record.
+                const recorded = item.status;
+                if (item.phase === "Started" || item.phase === "Terminal") {
+                    if (isFirstDecisive(recorded))
+                        yield* Deferred.succeed(item.firstDecisiveWaiter, recorded);
+                    yield* Deferred.succeed(item.startedWaiter, recorded);
+                }
+                if (item.phase === "Terminal")
+                    yield* Deferred.succeed(item.outcomeWaiter, recorded);
+                else
+                    yield* emit(item, item.phase === "Unknown" ||
+                        (item.clipId === undefined &&
+                            (item.unknownAtMs !== undefined || item.key === dispatching))
+                        ? { _tag: "Unknown", terminal: true }
+                        : { _tag: "Failed", reason: { _tag: "Scheduler", cause: terminal.value } }, true);
             }
-            else
-                yield* emit(item, { _tag: "Failed", reason: { _tag: "Scheduler", cause } });
+            const closed = Exit.isSuccess(terminal)
+                ? Exit.fail(PolicyFailure.refuse("SessionClosed", terminal.value.message))
+                : Exit.failCause(terminal.cause);
+            for (const [key, pending] of pendingWithdrawals) {
+                // A drop recorded before the claim is the withdrawal's actual result.
+                const reply = Exit.isSuccess(terminal) && items.get(key)?.status._tag === "Dropped"
+                    ? Exit.succeed("withdrawn")
+                    : closed;
+                for (const waiter of pending.replies)
+                    yield* Deferred.done(waiter, reply);
+            }
+            pendingWithdrawals.clear();
+            for (const reply of drainReplies.splice(0))
+                yield* Deferred.done(reply, closed);
+            yield* SubscriptionRef.update(stateRef, (state) => ({ ...state, accepting: false }));
+            yield* Deferred.done(initialized, Exit.isFailure(terminal)
+                ? Exit.failCause(terminal.cause)
+                : Exit.fail(ReactorError.fromCode("Closed", terminal.value.message)));
+            // Supervisors may close the owner immediately when this wakes them.
+            yield* Deferred.done(stopped, terminal);
+        }).pipe(Effect.onExit((exit) => Effect.gen(function* () {
+            // A bookkeeping defect must release joiners with that Cause, never
+            // leave an unfinished latch or report a partial settlement as success.
+            yield* Deferred.done(settled, exit);
+            if (Exit.isFailure(exit)) {
+                yield* Deferred.failCause(initialized, exit.cause);
+                yield* Deferred.failCause(stopped, exit.cause);
+            }
+        })));
+    }).pipe(Effect.uninterruptible);
+    const closeActor = (cause) => terminate(Exit.succeed(cause));
+    let recoveryState;
+    let capacityRecovery;
+    let recoveryDeadline;
+    let recoveryChanged = Deferred.makeUnsafe();
+    // Retain the original age while Closing hides the old source and acquisition
+    // is still pending. Only accepted evidence or a Ready replacement clears it.
+    const updateRecovery = (state = recoveryState, fresh) => {
+        if (ended !== undefined)
+            return;
+        recoveryState = state;
+        const preferred = state === undefined ? undefined : Option.getOrUndefined(state.preferredSessionId);
+        const ready = preferred !== undefined &&
+            state?.sessions.some((source) => source.sessionId === preferred && source.availability === "Ready") === true;
+        let earliest;
+        let drainDeadline;
+        const consider = (atMs, sessionId, cause, newlyObserved = false) => {
+            const deadlineMs = atMs + unknownRecoveryMs;
+            if (((!ready && newlyObserved) || sessionId === undefined || sessionId === preferred) &&
+                (earliest === undefined || deadlineMs < earliest.deadlineMs))
+                earliest = { deadlineMs, sessionId, cause };
+            if (draining && (drainDeadline === undefined || deadlineMs < drainDeadline.deadlineMs))
+                drainDeadline = { deadlineMs, sessionId, cause };
+        };
+        for (const entry of unknownFillers.values())
+            consider(entry.atMs, entry.sessionId, entry.cause);
+        for (const item of items.values())
+            if (item.phase !== "Terminal" &&
+                item.clipId === undefined &&
+                item.unknownAtMs !== undefined &&
+                item.unknownCause !== undefined)
+                consider(item.unknownAtMs, item.unknownSessionId, item.unknownCause);
+        if (fresh !== undefined)
+            consider(fresh.atMs, fresh.sessionId, fresh.cause, true);
+        if (ready &&
+            (earliest === undefined ||
+                (capacityRecovery?.sessionId !== undefined && capacityRecovery.sessionId !== preferred)))
+            capacityRecovery = undefined;
+        if (earliest !== undefined &&
+            (capacityRecovery === undefined || earliest.deadlineMs < capacityRecovery.deadlineMs))
+            capacityRecovery = earliest;
+        const next = drainDeadline !== undefined &&
+            (capacityRecovery === undefined || drainDeadline.deadlineMs < capacityRecovery.deadlineMs)
+            ? drainDeadline
+            : capacityRecovery;
+        if (next?.deadlineMs !== recoveryDeadline?.deadlineMs ||
+            next?.cause !== recoveryDeadline?.cause) {
+            recoveryDeadline = next;
+            Deferred.doneUnsafe(recoveryChanged, Exit.void);
         }
-        const closed = PolicyFailure.refuse("SessionClosed", cause.message);
-        for (const [key, pending] of pendingWithdrawals) {
-            for (const reply of pending.replies)
-                yield* Deferred.fail(reply, closed);
-            pendingWithdrawals.delete(key);
+    };
+    const provedUnknown = (atMs) => {
+        if (atMs !== undefined && capacityRecovery?.deadlineMs === atMs + unknownRecoveryMs)
+            capacityRecovery = undefined;
+    };
+    const watchdog = Effect.gen(function* () {
+        for (;;) {
+            // A committed result and its queued actor evidence can arrive in one
+            // turn. Coalesce those revisions before allocating a replacement timer;
+            // the original monotonic deadline remains unchanged.
+            yield* Effect.yieldNow;
+            if (ended !== undefined)
+                return;
+            recoveryChanged = Deferred.makeUnsafe();
+            const episode = recoveryDeadline;
+            if (episode === undefined) {
+                yield* Deferred.await(recoveryChanged);
+                continue;
+            }
+            yield* Deferred.await(recoveryChanged).pipe(Effect.timeoutOrElse({
+                duration: Math.max(0, episode.deadlineMs - monotonicMillis(clock)),
+                orElse: () => Effect.void,
+            }));
+            // A canceled timer may already have resumed. Recheck the retained
+            // identity without awaiting the actor, Engine state, or its permits.
+            if (ended === undefined &&
+                recoveryDeadline === episode &&
+                monotonicMillis(clock) >= episode.deadlineMs) {
+                const causes = new Set([episode.cause]);
+                for (const item of items.values())
+                    if (item.unknownCause !== undefined)
+                        causes.add(item.unknownCause);
+                for (const entry of unknownFillers.values())
+                    causes.add(entry.cause);
+                yield* closeActor(ReactorError.fromCode("Timeout", "Scheduler unknown recovery deadline elapsed", {
+                    operation: "scheduler.unknownRecovery",
+                    detail: [...causes],
+                }));
+                return;
+            }
         }
-        for (const reply of drainReplies.splice(0))
-            yield* Deferred.fail(reply, closed);
-        yield* SubscriptionRef.update(stateRef, (state) => ({ ...state, accepting: false }));
     });
     const prune = (state) => {
         const active = new Set(activeIds(state));
@@ -350,7 +521,44 @@ export const makeScheduler = (options) => Effect.gen(function* () {
         while (history.size > maxHistory)
             history.delete(history.keys().next().value);
     };
-    const publishState = (state) => {
+    // Capture membership and public rows in one pass. A fresh token reads every
+    // source on every observation, including in-place changes to Engine arrays;
+    // frozen empty rows can be shared by every published state without treating
+    // array identity as proof.
+    const noReady = Object.freeze([]);
+    const sourceRows = new Map();
+    const projectSources = (state) => {
+        const seen = Symbol();
+        const sessions = state.sessions.map(({ sessionId }) => {
+            let cached = sourceRows.get(sessionId);
+            if (cached === undefined) {
+                cached = { empty: Object.freeze({ sessionId, ready: noReady }), seen };
+                sourceRows.set(sessionId, cached);
+            }
+            else
+                cached.seen = seen;
+            if (state.ready.length === 0)
+                return cached.empty;
+            return {
+                sessionId,
+                ready: state.ready
+                    .filter((clip) => clip.sessionId === sessionId)
+                    .map((clip) => {
+                    const owner = owned.get(clip.clipId);
+                    return owner === undefined ? "other" : owner._tag === "Filler" ? "filler" : owner.key;
+                }),
+            };
+        });
+        // Bound cached rows by snapshot size without scanning unchanged overlap.
+        // Membership still requires this observation's token, even before pruning.
+        if (sourceRows.size > state.sessions.length)
+            for (const [id, cached] of sourceRows)
+                if (cached.seen !== seen)
+                    sourceRows.delete(id);
+        // Only the serialized actor uses this membership view, before its next projection.
+        return { sessions, has: (id) => sourceRows.get(id)?.seen === seen };
+    };
+    const publishState = (state, sessions = projectSources(state).sessions) => {
         const playing = Option.getOrUndefined(state.playing);
         const playingOwner = playing === undefined ? undefined : owned.get(playing.clipId);
         const playingKey = playing === undefined
@@ -372,15 +580,7 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                     .filter((item) => item.lane === name && item.phase !== "Terminal")
                     .map((item) => item.key),
             })),
-            sessions: state.sessions.map(({ sessionId }) => ({
-                sessionId,
-                ready: state.ready
-                    .filter((clip) => clip.sessionId === sessionId)
-                    .map((clip) => {
-                    const owner = owned.get(clip.clipId);
-                    return owner === undefined ? "other" : owner._tag === "Filler" ? "filler" : owner.key;
-                }),
-            })),
+            sessions,
             starved,
         });
     };
@@ -454,8 +654,8 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             filler.set(index, { index, clipId: record.clipId, sessionId: record.sessionId });
             const priorIndex = fillerIndex;
             fillerIndex = Math.max(fillerIndex, index + 1);
-            if (unknownFiller?.index === index)
-                unknownFiller = undefined;
+            provedUnknown(unknownFillers.get(index)?.atMs);
+            unknownFillers.delete(index);
             if (fillerIndex !== priorIndex)
                 upcomingFiller = undefined;
         }
@@ -467,6 +667,9 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                 if (resumed !== undefined) {
                     item.clipId = resumed.clipId;
                     item.sessionId = resumed.sessionId;
+                    provedUnknown(item.unknownAtMs);
+                    item.unknownAtMs = undefined;
+                    item.unknownCause = undefined;
                     item.unknownSessionId = undefined;
                     owned.set(resumed.clipId, { _tag: "Item", key: item.key });
                 }
@@ -530,7 +733,7 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                     lastProcessedSnapshotSerial > item.acknowledgedAtSnapshotSerial))
                 yield* emit(item, { _tag: "Unobserved" });
         }
-        yield* publishState(state);
+        updateRecovery(state);
     });
     const onEvent = (event) => Effect.gen(function* () {
         if (event._tag === "Starved")
@@ -662,6 +865,12 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                     if (command.sessionId === undefined ||
                         Option.getOrUndefined((yield* engine.state).preferredSessionId) !== command.sessionId)
                         return yield* PolicyFailure.refuse("RouteChanged", "The preferred source changed before dispatch");
+                    // Termination can be claimed while the route is read; its settlement
+                    // must not be followed by a new dispatch.
+                    if (ended !== undefined)
+                        return yield* PolicyFailure.refuse("SessionClosed", "The scheduler closed");
+                    if (command._tag === "Build")
+                        dispatching = command.key;
                     return yield* engine.enqueueOnSource === undefined
                         ? engine.enqueue(command.request)
                         : engine.enqueueOnSource(command.request, command.sessionId);
@@ -679,12 +888,44 @@ export const makeScheduler = (options) => Effect.gen(function* () {
     };
     const commandWorker = Effect.gen(function* () {
         for (;;) {
-            const command = yield* Queue.take(commandQueue);
+            const command = yield* takeQueue(commandQueue);
+            if (ended !== undefined)
+                continue;
             const result = yield* Effect.result(executeCommand(command));
+            dispatching = undefined;
+            // Capture uncertainty before actor delivery: drain may be awaiting a
+            // permit in stopRenewal while this committed command finishes.
+            if (ended === undefined &&
+                Result.isFailure(result) &&
+                result.failure.context.outcome === "unknown") {
+                let fresh;
+                const sessionId = engine.enqueueOnSource === undefined ||
+                    (command._tag !== "Build" && command._tag !== "BuildFiller")
+                    ? undefined
+                    : command.sessionId;
+                if (command._tag === "Build") {
+                    const item = items.get(command.key);
+                    if (item !== undefined && item.phase !== "Terminal" && item.clipId === undefined) {
+                        item.unknownAtMs ??= monotonicMillis(clock);
+                        item.unknownCause = result.failure;
+                        item.unknownSessionId = sessionId;
+                        fresh = { sessionId, atMs: item.unknownAtMs, cause: result.failure };
+                    }
+                }
+                else if (command._tag === "BuildFiller" &&
+                    !filler.has(command.index) &&
+                    !unknownFillers.has(command.index)) {
+                    fresh = { sessionId, atMs: monotonicMillis(clock), cause: result.failure };
+                    unknownFillers.set(command.index, { ...fresh, index: command.index });
+                }
+                updateRecovery(recoveryState, fresh);
+            }
             yield* Queue.offer(inbox, { _tag: "CommandDone", command, result });
         }
     });
     const sendCommand = (command) => Effect.gen(function* () {
+        if (ended !== undefined)
+            return;
         if (command._tag === "Build")
             pendingBuilds.add(command.key);
         if (command._tag === "DeferAt")
@@ -793,6 +1034,12 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                 break;
             }
             case "BuildFiller": {
+                if (unknownFillers.size >= maxUnknownFillers) {
+                    yield* closeActor(ReactorError.fromCode("Overflow", "Scheduler uncertain filler ledger is full", {
+                        operation: "scheduler.unknownRecovery",
+                    }));
+                    break;
+                }
                 const captured = upcomingFiller === undefined
                     ? yield* Effect.result(captureFiller(fillerIndex, runwaySeconds(state, monotonicMillis(clock), playingStartedMs, owned, [
                         ...items.values(),
@@ -862,6 +1109,8 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                     item.sessionId =
                         knownRecord(yield* engine.state, clipId)?.sessionId ??
                             (engine.enqueueOnSource === undefined ? undefined : command.sessionId);
+                    provedUnknown(item.unknownAtMs);
+                    item.unknownAtMs = undefined;
                     item.unknownSessionId = undefined;
                     item.unknownCause = undefined;
                     yield* replayEarlyClipEvents(clipId);
@@ -928,11 +1177,7 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                     yield* replayEarlyClipEvents(clipId);
                 }
                 else if (result.failure.context.outcome === "unknown") {
-                    if (!filler.has(command.index))
-                        unknownFiller = {
-                            index: command.index,
-                            sessionId: engine.enqueueOnSource === undefined ? undefined : command.sessionId,
-                        };
+                    // The worker already retained this identity and its original age.
                 }
                 else {
                     fillerRetryAtMs = monotonicMillis(clock) + 1_000;
@@ -1097,28 +1342,26 @@ export const makeScheduler = (options) => Effect.gen(function* () {
     };
     const reconcile = Effect.gen(function* () {
         const state = yield* engine.state;
+        if (ended !== undefined)
+            return;
         prune(state);
         refreshAnchors();
+        const sources = projectSources(state);
         if (!observationReady) {
-            yield* publishState(state);
+            yield* publishState(state, sources.sessions);
             return;
         }
-        if (unknownFiller !== undefined) {
-            const uncertain = unknownFiller;
-            const sourceRetired = uncertain.sessionId !== undefined &&
-                !state.sessions.some((session) => session.sessionId === uncertain.sessionId);
-            if (sourceRetired) {
-                unknownFiller = undefined;
-                fillerIndex = Math.max(fillerIndex, uncertain.index + 1);
-                upcomingFiller = undefined;
-            }
-            else if (uncertain.sessionId !== undefined &&
-                Option.getOrUndefined(state.preferredSessionId) !== uncertain.sessionId &&
-                fillerIndex <= uncertain.index) {
-                // A replacement may build independently, with a distinct key. The
-                // uncertain old request can still be adopted if it later appears.
-                fillerIndex = uncertain.index + 1;
-                upcomingFiller = undefined;
+        for (const uncertain of unknownFillers.values()) {
+            const sourceRetired = uncertain.sessionId !== undefined && !sources.has(uncertain.sessionId);
+            if (sourceRetired)
+                unknownFillers.delete(uncertain.index);
+            if (sourceRetired ||
+                (uncertain.sessionId !== undefined &&
+                    Option.getOrUndefined(state.preferredSessionId) !== uncertain.sessionId)) {
+                if (fillerIndex <= uncertain.index) {
+                    fillerIndex = uncertain.index + 1;
+                    upcomingFiller = undefined;
+                }
             }
         }
         for (const [key, pending] of pendingWithdrawals) {
@@ -1134,10 +1377,18 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             if (item.phase === "Unknown" &&
                 item.clipId === undefined &&
                 item.unknownSessionId !== undefined &&
-                !state.sessions.some((session) => session.sessionId === item.unknownSessionId))
+                !sources.has(item.unknownSessionId))
                 yield* resolveRetiredUnknown(item);
+        // After retirement, so a deadline no longer outlives the uncertainty that
+        // set it until the next message.
+        updateRecovery(state);
         if (draining)
             yield* drainPending(state);
+        const preferred = Option.getOrUndefined(state.preferredSessionId);
+        let unknownFillerCount = 0;
+        for (const entry of unknownFillers.values())
+            if (entry.sessionId === undefined || entry.sessionId === preferred)
+                unknownFillerCount++;
         const decision = plan({
             engine: state,
             items: [...items.values()],
@@ -1152,12 +1403,11 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             accepting: accepting || (draining && finishAccepted),
             fillerEnabled: !draining || fillerHeld(monotonicMillis(clock)),
             fillerRetryAtMs,
-            fillerUnknown: unknownFiller !== undefined,
-            fillerUnknownSessionId: unknownFiller?.sessionId,
+            unknownFillerCount,
             blockedMove,
         });
         refillActive = decision.refillActive;
-        yield* publishState(state);
+        yield* publishState(state, sources.sessions);
         if ((!draining || finishAccepted) && commandCount === 0 && decision.action !== undefined)
             yield* applyPolicy(decision.action, state);
         if (draining &&
@@ -1173,10 +1423,14 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             commandCount === 0 &&
             [...items.values()].every((item) => item.phase === "Terminal") &&
             pendingWithdrawals.size === 0 &&
-            unknownFiller === undefined) {
+            unknownFillers.size === 0) {
             const after = yield* engine.state;
             if (Option.isNone(after.playing) && !activeIds(after).some((clipId) => owned.has(clipId))) {
                 drained = true;
+                // A truthful completed drain no longer needs a replacement to resume
+                // service, even when retirement left a capacity episode behind.
+                capacityRecovery = undefined;
+                updateRecovery(after);
                 const replies = drainReplies.splice(0);
                 for (const reply of replies)
                     yield* Deferred.succeed(reply, undefined);
@@ -1185,16 +1439,10 @@ export const makeScheduler = (options) => Effect.gen(function* () {
     });
     const actor = Effect.gen(function* () {
         for (;;) {
-            const message = yield* Queue.take(inbox);
-            if (ended !== undefined && message._tag !== "CommandDone") {
-                if (message._tag === "Submit")
-                    yield* Deferred.fail(message.reply, PolicyFailure.refuse("SessionClosed", ended.message));
-                else if (message._tag === "Withdraw")
-                    yield* Deferred.fail(message.reply, PolicyFailure.refuse("SessionClosed", ended.message));
-                else if (message._tag === "Drain")
-                    yield* Deferred.fail(message.reply, PolicyFailure.refuse("SessionClosed", ended.message));
+            const message = yield* takeQueue(inbox);
+            // Queued public calls observe the same terminal result through closedCall.
+            if (ended !== undefined && message._tag !== "CommandDone")
                 continue;
-            }
             switch (message._tag) {
                 case "Submit": {
                     if (!accepting) {
@@ -1211,6 +1459,8 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                     }
                     const nowMs = monotonicMillis(clock);
                     const state = yield* engine.state;
+                    if (ended !== undefined)
+                        continue;
                     const startByMs = message.item.startByOffsetMs === undefined
                         ? undefined
                         : nowMs + message.item.startByOffsetMs;
@@ -1219,9 +1469,12 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                         yield* Deferred.fail(message.reply, WouldMissDeadline.of(message.item.key));
                         break;
                     }
-                    const startedWaiter = yield* Deferred.make();
-                    const outcomeWaiter = yield* Deferred.make();
-                    const firstDecisiveWaiter = yield* Deferred.make();
+                    // No yield from the terminal check above through admission and its
+                    // publication below: a claim either precedes the item or settles it
+                    // after its Accepted evidence.
+                    const startedWaiter = Deferred.makeUnsafe();
+                    const outcomeWaiter = Deferred.makeUnsafe();
+                    const firstDecisiveWaiter = Deferred.makeUnsafe();
                     const handle = {
                         key: message.item.key,
                         started: Deferred.await(startedWaiter),
@@ -1251,18 +1504,22 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                         sessionId: undefined,
                         unknownSessionId: undefined,
                         unknownCause: undefined,
+                        unknownAtMs: undefined,
                         handle,
                         startedWaiter,
                         outcomeWaiter,
                         firstDecisiveWaiter,
                     };
                     items.set(item.key, item);
-                    yield* PubSub.publish(events, {
+                    PubSub.publishUnsafe(events, {
                         key: item.key,
                         at: clock.currentTimeMillisUnsafe(),
                         status: item.status,
                     });
                     yield* observed(state);
+                    // Submit returns its committed view; other messages publish once
+                    // through reconciliation after all their evidence is adopted.
+                    yield* publishState(state);
                     yield* Deferred.succeed(message.reply, handle);
                     break;
                 }
@@ -1288,9 +1545,12 @@ export const makeScheduler = (options) => Effect.gen(function* () {
                     if (!draining) {
                         draining = true;
                         finishAccepted = message.finish === "accepted";
+                        updateRecovery();
                         // Renewal admission must stop even while an earlier enqueue is
                         // awaiting its reply in the command worker.
                         const stoppedRenewal = yield* Effect.result(engine.stopRenewal);
+                        if (ended !== undefined)
+                            break;
                         if (Result.isFailure(stoppedRenewal)) {
                             drainFailure = stoppedRenewal.failure;
                             for (const reply of drainReplies.splice(0))
@@ -1362,22 +1622,12 @@ export const makeScheduler = (options) => Effect.gen(function* () {
             }
         }
     });
-    // A defect remains a defect, but it must wake waiters just as a typed
-    // terminal failure does. Scope interruption uses the finalizer below.
+    // Orderly scope teardown has already claimed and settled Closed before
+    // worker interruption; unexpected interruption retains its original Cause.
     const supervise = (effect) => effect.pipe(Effect.onExit((exit) => Exit.isFailure(exit) && ended === undefined
-        ? Effect.gen(function* () {
-            accepting = false;
-            yield* Deferred.failCause(stopped, exit.cause);
-            yield* Deferred.failCause(initialized, exit.cause);
-            for (const item of items.values()) {
-                yield* Deferred.failCause(item.startedWaiter, exit.cause);
-                yield* Deferred.failCause(item.outcomeWaiter, exit.cause);
-                yield* Deferred.failCause(item.firstDecisiveWaiter, exit.cause);
-            }
-            yield* Queue.shutdown(inbox);
-            yield* Queue.shutdown(commandQueue);
-        })
+        ? terminate(Exit.failCause(exit.cause))
         : Effect.void));
+    yield* Effect.forkScoped(supervise(watchdog));
     yield* Effect.forkScoped(supervise(commandWorker));
     yield* Effect.forkScoped(supervise(actor));
     yield* Effect.forkScoped(supervise(observe));
@@ -1425,6 +1675,6 @@ export const makeScheduler = (options) => Effect.gen(function* () {
         state: SubscriptionRef.get(stateRef),
         asRun: Stream.fromPubSub(events),
     };
-});
+}).pipe(Effect.provideServiceEffect(Scope.Scope, Effect.flatMap(Scope.Scope, (parent) => Scope.fork(parent, "sequential"))));
 export const layerScheduler = (options) => Layer.effect(Scheduler, makeScheduler(options));
 //# sourceMappingURL=scheduler.js.map