reactor-effect-client 0.2.0 → 0.3.0

package/dist/h3/_internal/client.js

@@ -6,87 +6,82 @@ import * as Result from "effect/Result";
 import * as Scope from "effect/Scope";
 import * as Semaphore from "effect/Semaphore";
 import * as Stream from "effect/Stream";
-import { errorOf, positiveLimit, ReactorError } from "../../errors.js";
+import { CommandFailure, parsed, PolicyFailure, positiveLimit, ReactorError, Remote, } from "../../errors.js";
+import { duration } from "../../duration.js";
 import { Observations } from "../../observation.js";
-import { CommandFailure } from "../../session/commands.js";
 import * as Submission from "../../Submission.js";
 import { decodeMessage } from "../messages.js";
+import { Operations } from "./operations.js";
 import { canvases } from "../profile.js";
 import { Commands } from "./contracts.js";
 import { validateDeployment } from "./deployment.js";
 import { acceptanceFor, encodeMetadata, submissionFromMetadata } from "./evidence.js";
 import { checkedUpload, referenceMaterial } from "./references.js";
 import { captureRequest, clipId, enqueueArguments, nonnegative, seconds } from "./request.js";
+import { sizeOf } from "./retained.js";
 import { ProviderState } from "./state.js";
-const pure = (evaluate) => Effect.try({ try: evaluate, catch: errorOf });
-const localFailure = (operation, cause) => new CommandFailure(cause, { ...cause.context, operation, outcome: "not-submitted" });
-const uncertain = (operation, source, message, cause) => new CommandFailure(cause ?? new ReactorError("UnexpectedReply", message), {
+const pure = (evaluate) => parsed(evaluate);
+const localFailure = (operation, cause) => CommandFailure.from(cause, { ...cause.context, operation, outcome: "not-submitted" });
+/** A caller's local refusal already proves no dispatch; anything else becomes one. */
+const preparationFailure = (operation, cause) => ReactorError.is(cause) || CommandFailure.is(cause) ? localFailure(operation, cause) : cause;
+const uncertain = (operation, source, message, cause) => CommandFailure.from(cause ?? ReactorError.fromCode("UnexpectedReply", message), {
     ...(cause?.context ?? {}),
     operation,
     outcome: "unknown",
     requestId: source.requestId,
     generation: source.generation,
 });
-const rejected = (operation, source) => new CommandFailure(new ReactorError("Remote", `H3 ${operation} was refused`), {
-    operation,
-    outcome: "replied",
-    requestId: source.requestId,
-    generation: source.generation,
+/** The refusal reason is provider free text with no stable codes: kept for diagnosis only. */
+const rejected = (operation, source, reason) => new CommandFailure({
+    reason: new Remote({ _tag: "Remote", message: `H3 ${operation} was refused`, body: reason }),
+    context: {
+        operation,
+        outcome: "replied",
+        requestId: source.requestId,
+        generation: source.generation,
+    },
 });
 const hex = (bytes) => Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
-/** Conservative retained-size accounting includes shared objects only once. */
-const sizeOf = (input) => {
-    const pending = [input], seen = new Set();
-    let bytes = 0, nodes = 0;
-    while (pending.length > 0) {
-        if (++nodes > 1_000_000)
-            return Number.MAX_SAFE_INTEGER;
-        const value = pending.pop();
-        if (typeof value === "string")
-            bytes += value.length * 3;
-        else if (value !== null && typeof value === "object") {
-            if (seen.has(value))
-                continue;
-            seen.add(value);
-            if (ArrayBuffer.isView(value))
-                bytes += value.byteLength;
-            else if (value instanceof ArrayBuffer)
-                bytes += value.byteLength;
-            else if (value instanceof Map) {
-                bytes += value.size * 32;
-                for (const [key, child] of value) {
-                    pending.push(key);
-                    pending.push(child);
-                }
-            }
-            else if (value instanceof Set) {
-                bytes += value.size * 16;
-                for (const child of value)
-                    pending.push(child);
-            }
-            else {
-                bytes += 32;
-                for (const child of Object.values(value))
-                    pending.push(child);
-            }
-        }
-        else
-            bytes += 8;
+/**
+ * Whether a snapshot at `revision` already reflects `event`: the reducer
+ * handles session events in sequence order. An acceptance is local evidence
+ * that no snapshot holds, and a diagnostic without a source has no sequence,
+ * so neither is ever covered.
+ */
+const covered = (event, revision) => {
+    switch (event._tag) {
+        case "Acceptance":
+            return false;
+        case "Diagnostic":
+            return event.source !== undefined && event.source.sequence <= revision;
+        default:
+            return event.source.sequence <= revision;
     }
-    return bytes;
 };
 const build = (session, options) => Effect.gen(function* () {
     const scope = yield* Effect.scope;
     const crypto = yield* Crypto.Crypto;
     const limits = yield* pure(() => ({
-        command: positiveLimit(options.commandTimeoutMs ?? 15000, "H3 command deadline", 600000),
-        setup: positiveLimit(options.setupTimeoutMs ?? 60000, "H3 setup deadline", 600000),
-        reconcile: positiveLimit(options.reconcileWindowMs ?? 5000, "H3 reconciliation deadline", 60000),
-        hook: positiveLimit(options.resultHookTimeoutMs ?? 1000, "H3 result hook deadline", 60000),
+        command: duration(options.replyTimeout ?? "15 seconds", "H3 reply timeout", {
+            maximum: "10 minutes",
+        }),
+        upload: duration(options.uploadTimeout ?? "60 seconds", "H3 upload timeout", {
+            maximum: "10 minutes",
+        }),
+        setup: duration(options.setupTimeout ?? "60 seconds", "H3 setup timeout", {
+            maximum: "10 minutes",
+        }),
+        reconcile: duration(options.reconcileWindow ?? "5 seconds", "H3 reconciliation window", {
+            maximum: "1 minute",
+        }),
+        hook: duration(options.resultHookTimeout ?? "1 second", "H3 result hook timeout", {
+            maximum: "1 minute",
+        }),
         pending: positiveLimit(options.maxPending ?? 128, "H3 pending acceptance bound", 4096),
         clips: positiveLimit(options.maxTrackedClips ?? 4096, "H3 clip observation bound", 16384),
         acceptances: positiveLimit(options.maxAcceptances ?? 1024, "H3 local acceptance bound", 16384),
         cache: positiveLimit(options.maxCachedUploads ?? 256, "H3 upload cache bound", 4096),
+        operations: positiveLimit(options.maxOperations ?? 1024, "H3 clip operation bound", 16384),
         prompt: positiveLimit(options.maxPromptBytes ?? 1048576, "H3 prompt byte bound", 4194304),
         retained: positiveLimit(options.maxRetainedBytes ?? 16777216, "H3 retained byte bound", 67108864),
     }));
@@ -95,7 +90,7 @@ const build = (session, options) => Effect.gen(function* () {
     yield* session.ready;
     const observation = yield* session.observe(options.observation);
     const state = new ProviderState(session.id, observation.initial.generation, observation.revision, limits.clips);
-    const namespace = hex(yield* crypto.randomBytes(16).pipe(Effect.mapError(() => new ReactorError("InvalidState", "Could not allocate H3 acceptance namespace", {
+    const namespace = hex(yield* crypto.randomBytes(16).pipe(Effect.mapError(() => ReactorError.fromCode("InvalidState", "Could not allocate H3 acceptance namespace", {
         outcome: "not-submitted",
     }))));
     let counter = 0n, closed = false, fatalError;
@@ -103,26 +98,83 @@ const build = (session, options) => Effect.gen(function* () {
     const events = new Observations();
     const pending = new Map();
     const acceptances = new Map();
+    /** Acceptances count toward the reducer's retained bytes while the map holds them. */
+    const retain = (acceptance, direction) => {
+        state.retained.add(direction * (64 + acceptance.submissionId.length * 3));
+        if (direction === 1) {
+            state.retained.hold(acceptance.clip);
+            state.retained.hold(acceptance.evidence.source);
+        }
+        else {
+            state.retained.drop(acceptance.clip);
+            state.retained.drop(acceptance.evidence.source);
+        }
+    };
+    const operations = new Operations(limits.operations);
     const decoded = new WeakMap();
     const observedWaiters = new Map();
     const uploads = new Map();
     const uploadGate = yield* Semaphore.make(1);
+    // Callers waiting for the provider to synchronize; every applied event and
+    // any failure wakes them to look again. A woken caller may wait again at
+    // once, so the set is emptied before any waiter is released.
+    const synchronizing = new Set();
+    const wake = () => {
+        const waiters = [...synchronizing];
+        synchronizing.clear();
+        for (const waiter of waiters)
+            Deferred.doneUnsafe(waiter, Effect.void);
+    };
     const emit = (event) => events.emit(event, sizeOf(event));
+    /** Decide a submission's acceptance: record it, resolve its waiters and announce it. */
+    const record = (entry, acceptance) => {
+        entry.held = undefined;
+        if (acceptances.size >= limits.acceptances) {
+            const first = acceptances.entries().next();
+            if (!first.done) {
+                acceptances.delete(first.value[0]);
+                retain(first.value[1], -1);
+            }
+        }
+        acceptances.set(entry.id, acceptance);
+        retain(acceptance, 1);
+        operations.accept(acceptance);
+        Deferred.doneUnsafe(entry.deferred, Effect.succeed(acceptance));
+        emit({ _tag: "Acceptance", acceptance });
+    };
+    /**
+     * Held evidence decides the acceptance once the reply cannot: it came
+     * without correlating the clip, or the wait for it ended short of a
+     * definite failure, which discards the evidence instead.
+     */
+    const decideHeld = (entry) => {
+        if (entry.held !== undefined)
+            record(entry, entry.held);
+    };
     const fail = (error) => {
         if (fatalError !== undefined)
             return;
         fatalError = error;
         state.unavailable(error);
+        // Recording held evidence resumes its waiters at once: by now the provider
+        // refuses their new work, and they can read the acceptance before anyone
+        // learns of the failure.
+        for (const entry of pending.values())
+            decideHeld(entry);
         Deferred.doneUnsafe(fatal, Effect.succeed(error));
         for (const entry of pending.values())
             Deferred.doneUnsafe(entry.deferred, Effect.fail(error));
         for (const waiter of observedWaiters.values())
             Deferred.doneUnsafe(waiter, Effect.fail(error));
         emit({ _tag: "Diagnostic", error });
+        wake();
     };
     yield* Effect.addFinalizer(() => Effect.sync(() => {
         closed = true;
-        fail(new ReactorError("Closed", "H3 provider scope closed", { operation: "H3 observation" }));
+        fail(ReactorError.fromCode("Closed", "H3 provider scope closed", {
+            operation: "H3 observation",
+        }));
+        operations.retire();
         events.end();
         pending.clear();
         observedWaiters.clear();
@@ -133,41 +185,47 @@ const build = (session, options) => Effect.gen(function* () {
         if (id === undefined)
             return;
         const entry = pending.get(id);
-        if (entry === undefined)
-            return;
-        const acceptance = acceptanceFor(entry, clip, source);
-        if (acceptance === undefined)
+        const acceptance = entry === undefined ? undefined : acceptanceFor(entry, clip, source);
+        if (entry === undefined || acceptance === undefined) {
+            // After the reconcile window, or in a later transport generation, the
+            // evidence can still resolve the operation, never the acceptances.
+            operations.lateEvidence(id, clip, source);
             return;
-        const previous = acceptances.get(entry.id);
-        if (previous !== undefined) {
-            if (previous.clip.clip_id !== clip.clip_id)
-                fail(new ReactorError("Protocol", "H3 acceptance identity named two clips", {
-                    operation: "enqueue",
-                }));
+        }
+        const previous = acceptances.get(entry.id) ?? entry.held;
+        if (previous !== undefined && previous.clip.clip_id !== clip.clip_id) {
+            fail(ReactorError.fromCode("Protocol", "H3 acceptance identity named two clips", {
+                operation: "enqueue",
+            }));
             return;
         }
-        if (acceptances.size >= limits.acceptances) {
-            const first = acceptances.keys().next();
-            if (!first.done)
-                acceptances.delete(first.value);
+        if (acceptances.has(entry.id))
+            return;
+        // Until the reply is observed, it may still correlate the acceptance; the
+        // clip's facts are recorded meanwhile.
+        if (acceptance.evidence.kind === "metadata" && entry.awaiting) {
+            if (entry.held === undefined) {
+                entry.held = acceptance;
+                operations.identify(acceptance);
+            }
+            return;
         }
-        acceptances.set(entry.id, acceptance);
-        Deferred.doneUnsafe(entry.deferred, Effect.succeed(acceptance));
-        emit({ _tag: "Acceptance", acceptance });
+        record(entry, acceptance);
     };
     const reduce = (source) => {
         if (closed || fatalError !== undefined)
             return;
         try {
-            const before = state.snapshot().transportGeneration;
+            const before = state.transportGeneration;
             let disposition = state.admit(source);
-            if (state.snapshot().transportGeneration !== before) {
+            if (state.transportGeneration !== before) {
                 uploads.clear();
                 for (const entry of pending.values())
-                    if (entry.generation !== source.generation)
-                        Deferred.doneUnsafe(entry.deferred, Effect.fail(new ReactorError("Disconnected", "H3 acceptance crossed a transport generation", {
-                            operation: "enqueue",
-                        })));
+                    if (entry.generation !== source.generation) {
+                        // No reply to an earlier generation's enqueue can correlate now.
+                        decideHeld(entry);
+                        Deferred.doneUnsafe(entry.deferred, Effect.fail(ReactorError.fromCode("Disconnected", "H3 acceptance crossed a transport generation", { operation: "enqueue" })));
+                    }
             }
             if (source._tag !== "Model") {
                 emit({ _tag: "Session", source });
@@ -177,7 +235,7 @@ const build = (session, options) => Effect.gen(function* () {
             if (source.kind === "ack") {
                 result =
                     disposition === "stale"
-                        ? Result.fail(new ReactorError("Disconnected", "H3 received a stale command acknowledgement"))
+                        ? Result.fail(ReactorError.fromCode("Disconnected", "H3 received a stale command acknowledgement"))
                         : Result.succeed(undefined);
                 emit({ _tag: "Acknowledged", source });
             }
@@ -187,7 +245,7 @@ const build = (session, options) => Effect.gen(function* () {
                     disposition = state.apply(message, source);
                 result =
                     disposition === "stale"
-                        ? Result.fail(new ReactorError("Disconnected", "H3 received a stale command message"))
+                        ? Result.fail(ReactorError.fromCode("Disconnected", "H3 received a stale command message"))
                         : Result.succeed(message);
                 if (disposition !== "stale" && message.type !== "unknown") {
                     if (message.type === "queue_update") {
@@ -200,6 +258,7 @@ const build = (session, options) => Effect.gen(function* () {
                     }
                     else if ("clip" in message.data)
                         accept(message.data.clip, source);
+                    operations.observe(message, source);
                 }
                 emit({ _tag: "Message", message, source, disposition });
             }
@@ -207,14 +266,17 @@ const build = (session, options) => Effect.gen(function* () {
             const waiter = observedWaiters.get(source);
             if (waiter !== undefined)
                 Deferred.doneUnsafe(waiter, Result.isSuccess(result) ? Effect.succeed(result.success) : Effect.fail(result.failure));
-            if (sizeOf({ snapshot: state.snapshot(), acceptances: [...acceptances.values()] }) >
-                limits.retained)
-                fail(new ReactorError("Overflow", "H3 retained observation byte bound exceeded", {
+            if (state.retained.bytes > limits.retained)
+                fail(ReactorError.fromCode("Overflow", "H3 retained observation byte bound exceeded", {
                     operation: "H3 observation",
                 }));
         }
         catch (cause) {
-            const error = errorOf(cause);
+            // The reducer rejects provider data with a ReactorError; anything else
+            // it throws is a bug, which stays a defect of the observation fiber.
+            if (!ReactorError.is(cause))
+                throw cause;
+            const error = cause;
             if (source._tag === "Model") {
                 decoded.set(source, Result.fail(error));
                 const waiter = observedWaiters.get(source);
@@ -224,19 +286,50 @@ const build = (session, options) => Effect.gen(function* () {
             fail(error);
         }
     };
-    yield* observation.events.pipe(Stream.runForEach((source) => Effect.sync(() => reduce(source))), Effect.catch((error) => Effect.sync(() => fail(error))), Effect.andThen(Effect.sync(() => {
+    yield* observation.events.pipe(Stream.runForEach((source) => Effect.sync(() => {
+        reduce(source);
+        wake();
+    })), Effect.catch((error) => Effect.sync(() => fail(error))), Effect.andThen(Effect.sync(() => {
         if (!closed)
-            fail(new ReactorError("Closed", "H3 session observation ended"));
+            fail(ReactorError.fromCode("Closed", "H3 session observation ended"));
     })), Effect.forkScoped);
-    const active = (operation, needsFacts) => Effect.suspend(() => {
-        const current = state.snapshot();
-        const error = closed
-            ? new ReactorError("Closed", "H3 provider scope is closed")
-            : (fatalError ??
-                (needsFacts && current._tag !== "Ready"
-                    ? new ReactorError("InvalidState", "H3 provider needs current state and queue observations")
-                    : undefined));
-        return error === undefined ? Effect.void : Effect.fail(localFailure(operation, error));
+    // Enqueues execute in a scope of their own. Finalizers run last-added first,
+    // so a closing provider refuses new work before that scope interrupts them.
+    const executions = yield* Scope.fork(scope);
+    yield* Effect.addFinalizer(() => Effect.sync(() => {
+        closed = true;
+    }));
+    const refusal = (needsFacts) => closed
+        ? ReactorError.fromCode("Closed", "H3 provider scope is closed")
+        : (fatalError ??
+            (needsFacts && state.availability !== "Ready"
+                ? ReactorError.fromCode("InvalidState", "H3 provider needs current state and queue observations")
+                : undefined));
+    /**
+     * H3 documents that a command replies before it broadcasts the state and
+     * queue it changed (hosted H3 broadcasts an enqueue's queue first), so the
+     * provider may be briefly synchronizing after a reply. A command that needs
+     * current facts waits, within its own reply deadline, for those
+     * broadcasts; it is refused, not submitted, only if they do not come or
+     * the provider fails.
+     */
+    const synchronized = Effect.gen(function* () {
+        while (!closed && fatalError === undefined && state.availability === "Synchronizing") {
+            const waiter = Deferred.makeUnsafe();
+            synchronizing.add(waiter);
+            yield* Deferred.await(waiter).pipe(Effect.ensuring(Effect.sync(() => synchronizing.delete(waiter))));
+        }
+    });
+    /** Wait, within one reply deadline, while the provider is only synchronizing. */
+    const settled = Effect.suspend(() => refusal(false) === undefined && state.availability === "Synchronizing"
+        ? synchronized.pipe(Effect.timeout(limits.command), Effect.ignore)
+        : Effect.void);
+    const active = (operation, needsFacts) => Effect.gen(function* () {
+        if (needsFacts)
+            yield* settled;
+        const error = refusal(needsFacts);
+        if (error !== undefined)
+            return yield* localFailure(operation, error);
     });
     const awaitObservation = (source) => Effect.suspend(() => {
         const previous = decoded.get(source);
@@ -247,28 +340,31 @@ const build = (session, options) => Effect.gen(function* () {
         if (fatalError !== undefined)
             return Effect.fail(fatalError);
         if (observedWaiters.size >= limits.pending)
-            return Effect.fail(new ReactorError("Overflow", "H3 reply observer bound exceeded"));
+            return Effect.fail(ReactorError.fromCode("Overflow", "H3 reply observer bound exceeded"));
         const waiter = Deferred.makeUnsafe();
         observedWaiters.set(source, waiter);
         return Deferred.await(waiter).pipe(Effect.timeoutOrElse({
             duration: limits.command,
-            orElse: () => Effect.fail(new ReactorError("Timeout", "H3 did not observe the command's exact returned envelope")),
+            orElse: () => Effect.fail(ReactorError.fromCode("Timeout", "H3 did not observe the command's exact returned envelope")),
         }), Effect.ensuring(Effect.sync(() => {
             observedWaiters.delete(source);
         })));
     });
     const call = (operation, args, needsFacts = true) => Effect.gen(function* () {
         yield* active(operation, needsFacts);
-        const source = yield* session.command(operation, args, undefined, limits.command);
+        const source = yield* session.command(operation, args, { replyTimeout: limits.command });
         const message = yield* awaitObservation(source).pipe(Effect.mapError((error) => uncertain(operation, source, "H3 reply observation failed", error)));
         if (message?.type === "command_error" && message.data.command === operation)
-            return yield* Effect.fail(rejected(operation, source));
+            return yield* rejected(operation, source, message.data.reason);
         return { source, message };
     });
     const named = (operation, args, needsFacts = true) => {
         const expected = Commands[operation].reply;
         return call(operation, args, needsFacts).pipe(Effect.flatMap(({ source, message }) => message?.type === expected
-            ? Effect.succeed(Object.freeze({ value: message.data, source }))
+            ? // The broadcasts a reply implies follow it: the command settles once
+                // they are observed, so its caller reads its own effects. The state
+                // and queue reads are themselves that barrier and do not wait.
+                (needsFacts ? settled : Effect.void).pipe(Effect.as(Object.freeze({ value: message.data, source })))
             : Effect.fail(uncertain(operation, source, `H3 ${operation} did not return ${expected}`))));
     };
     const checked = (operation, value) => pure(value).pipe(Effect.mapError((error) => localFailure(operation, error)));
@@ -276,7 +372,7 @@ const build = (session, options) => Effect.gen(function* () {
     const getQueue = named("get_queue", {}, false);
     const refresh = getState.pipe(Effect.andThen(getQueue), Effect.flatMap((reply) => state.snapshot()._tag === "Ready"
         ? Effect.void
-        : Effect.fail(new CommandFailure(new ReactorError("InvalidState", "H3 full snapshots changed during refresh"), {
+        : Effect.fail(CommandFailure.from(ReactorError.fromCode("InvalidState", "H3 full snapshots changed during refresh"), {
             operation: "H3 refresh",
             outcome: "replied",
             requestId: reply.source.requestId,
@@ -285,13 +381,13 @@ const build = (session, options) => Effect.gen(function* () {
     const control = (operation, args) => call(operation, args).pipe(Effect.flatMap(({ source, message }) => source.kind === "ack"
         ? Effect.succeed(Object.freeze({ _tag: "Acknowledged", source }))
         : message !== undefined && message.type !== "unknown"
-            ? Effect.succeed(Object.freeze({ _tag: "Reply", message, source }))
+            ? settled.pipe(Effect.as(Object.freeze({ _tag: "Reply", message, source })))
             : Effect.fail(uncertain(operation, source, "Unexpected H3 control response"))));
     const stage = (request, metadata) => Effect.gen(function* () {
         yield* active("enqueue", true);
         const before = state.snapshot();
         if (before._tag !== "Ready")
-            return yield* Effect.fail(localFailure("enqueue", new ReactorError("InvalidState", "H3 is not ready")));
+            return yield* localFailure("enqueue", ReactorError.fromCode("InvalidState", "H3 is not ready"));
         if (request.seconds !== undefined)
             yield* checked("enqueue", () => seconds(request.seconds, {
                 min: before.state.clip_seconds_min,
@@ -306,20 +402,22 @@ const build = (session, options) => Effect.gen(function* () {
             }
             const digest = yield* crypto
                 .digest("SHA-256", material.bytes)
-                .pipe(Effect.mapError(() => localFailure("enqueue", new ReactorError("InvalidState", "Could not hash H3 reference"))));
+                .pipe(Effect.mapError(() => localFailure("enqueue", ReactorError.fromCode("InvalidState", "Could not hash H3 reference"))));
             const key = `${before.transportGeneration}:${hex(digest)}`;
             const file = yield* uploadGate.withPermit(Effect.gen(function* () {
                 const cached = uploads.get(key);
                 if (cached !== undefined)
                     return cached;
                 const uploaded = yield* session
-                    .upload(`h3-${hex(digest)}`, reference.mimeType, material.bytes, limits.command)
+                    .upload(`h3-${hex(digest)}`, reference.mimeType, material.bytes, {
+                    uploadTimeout: limits.upload,
+                })
                     .pipe(Effect.mapError((error) => localFailure("enqueue", error)));
                 const file = yield* checked("enqueue", () => checkedUpload(uploaded.file));
                 if (file.size !== BigInt(reference.size) || file.mime_type !== reference.mimeType)
-                    return yield* Effect.fail(localFailure("enqueue", new ReactorError("Protocol", "H3 upload returned different file facts")));
+                    return yield* localFailure("enqueue", ReactorError.fromCode("Protocol", "H3 upload returned different file facts"));
                 if (state.snapshot().transportGeneration !== before.transportGeneration)
-                    return yield* Effect.fail(localFailure("enqueue", new ReactorError("Disconnected", "H3 reference upload crossed a generation")));
+                    return yield* localFailure("enqueue", ReactorError.fromCode("Disconnected", "H3 reference upload crossed a generation"));
                 if (uploads.size >= limits.cache) {
                     const first = uploads.keys().next();
                     if (!first.done)
@@ -335,7 +433,7 @@ const build = (session, options) => Effect.gen(function* () {
     });
     const nextId = () => pure(() => {
         if (counter === 0xffffffffffffffffn)
-            throw new ReactorError("Overflow", "H3 submission identity space exhausted", {
+            throw ReactorError.fromCode("Overflow", "H3 submission identity space exhausted", {
                 outcome: "not-submitted",
             });
         return `${namespace}:${++counter}`;
@@ -343,27 +441,29 @@ const build = (session, options) => Effect.gen(function* () {
     const prepared = (id, input, hooks) => Submission.make({
         id,
         prepare: Effect.gen(function* () {
-            const { request, metadata } = yield* input.pipe(Effect.mapError((error) => localFailure("enqueue", error)));
+            const { request, metadata } = yield* input.pipe(Effect.mapError((error) => preparationFailure("enqueue", error)));
             const staged = yield* stage(request, metadata);
-            return {
-                ...staged,
-                entry: {
-                    id,
-                    metadata,
-                    prompt: request.prompt,
-                    generation: staged.generation,
-                    deferred: Deferred.makeUnsafe(),
-                },
+            const entry = {
+                id,
+                metadata,
+                prompt: request.prompt,
+                generation: staged.generation,
+                deferred: Deferred.makeUnsafe(),
+                awaiting: true,
+                held: undefined,
             };
+            return { ...staged, entry };
         }),
         commit: ({ entry }) => Effect.gen(function* () {
             yield* active("enqueue", true);
             const live = yield* session.ready.pipe(Effect.mapError((error) => localFailure("enqueue", error)));
             if (entry.generation !== state.snapshot().transportGeneration ||
                 live.generation !== entry.generation)
-                return yield* Effect.fail(localFailure("enqueue", new ReactorError("Disconnected", "H3 prepared input crossed a transport generation")));
+                return yield* localFailure("enqueue", ReactorError.fromCode("Disconnected", "H3 prepared input crossed a transport generation"));
             if (pending.size >= limits.pending)
-                return yield* Effect.fail(localFailure("enqueue", new ReactorError("Overflow", "H3 pending acceptance bound reached")));
+                return yield* localFailure("enqueue", ReactorError.fromCode("Overflow", "H3 pending acceptance bound reached"));
+            // The operation's slot is taken before anything is sent.
+            yield* pure(() => operations.reserve(entry)).pipe(Effect.mapError((error) => localFailure("enqueue", error)));
             // Acceptance registration is inside the same commit boundary as the
             // orchestration hook and precedes Session.command's wire correlation.
             pending.set(id, entry);
@@ -371,16 +471,17 @@ const build = (session, options) => Effect.gen(function* () {
                 yield* hooks.commit(id).pipe(Effect.onExit((exit) => Exit.isFailure(exit)
                     ? Effect.sync(() => {
                         pending.delete(id);
+                        operations.abandon(id);
                     })
                     : Effect.void));
         }),
         execute: ({ args, entry }) => {
             const execution = Effect.gen(function* () {
-                const sent = yield* Effect.result(session.command("enqueue", args, undefined, limits.command));
+                const sent = yield* Effect.result(session.command("enqueue", args, { replyTimeout: limits.command }));
                 let original;
                 if (Result.isFailure(sent)) {
                     if (sent.failure.context.outcome !== "unknown")
-                        return yield* Effect.fail(sent.failure);
+                        return yield* sent.failure;
                     original = sent.failure;
                 }
                 else {
@@ -389,29 +490,58 @@ const build = (session, options) => Effect.gen(function* () {
                     if (Result.isSuccess(observed) &&
                         observed.success?.type === "command_error" &&
                         observed.success.data.command === "enqueue")
-                        return yield* Effect.fail(rejected("enqueue", source));
+                        return yield* rejected("enqueue", source, observed.success.data.reason);
                     original = uncertain("enqueue", source, "H3 enqueue has no proven clip acceptance", Result.isFailure(observed) ? observed.failure : undefined);
                 }
-                return yield* Deferred.await(entry.deferred).pipe(Effect.timeoutOrElse({
+                // The reply was observed or is lost: evidence by metadata that came
+                // while it was awaited now decides, and later evidence at once.
+                entry.awaiting = false;
+                decideHeld(entry);
+                const acceptance = yield* Deferred.await(entry.deferred).pipe(Effect.timeoutOrElse({
                     duration: limits.reconcile,
                     orElse: () => Effect.fail(original),
-                }), Effect.mapError(() => original));
-            }).pipe(Effect.ensuring(Effect.sync(() => {
+                }), Effect.mapError(() => original), Effect.withSpan("reactor.h3.reconcile", {}, { captureStackTrace: false }));
+                // Like every command, an enqueue settles once the snapshots its reply
+                // implies are observed, so its caller finds the clip in the queue.
+                yield* settled;
+                return acceptance;
+            }).pipe(Effect.onExit((exit) => Effect.sync(() => {
+                // However the enqueue ends, interrupted as the provider closes or
+                // otherwise, the evidence it saw still decides; only a definite
+                // failure discards it.
+                const error = Exit.findError(exit);
+                if (!(error._tag === "Success" && error.success.context.outcome !== "unknown"))
+                    decideHeld(entry);
                 pending.delete(id);
-            })));
+            })), Effect.onExit((exit) => Effect.sync(() => {
+                operations.settle(id, exit);
+            })), 
+            // The submission's own execution fiber carries the span, so it ends with
+            // the enqueue's outcome even when the caller stopped waiting.
+            Effect.onExit((exit) => {
+                const error = Exit.findError(exit);
+                return Effect.annotateCurrentSpan(Exit.isSuccess(exit)
+                    ? { "reactor.command.outcome": "replied" }
+                    : error._tag === "Success"
+                        ? {
+                            "reactor.command.outcome": error.success.context.outcome,
+                            "error.type": error.success.reason._tag,
+                        }
+                        : {});
+            }), Effect.withSpan("reactor.h3.enqueue", { kind: "client", attributes: { "reactor.h3.submission.id": id } }, { captureStackTrace: false }));
             return hooks.result === undefined
                 ? execution
                 : Effect.result(execution).pipe(Effect.flatMap((result) => Effect.gen(function* () {
                     const hook = yield* Effect.exit(Effect.suspend(() => hooks.result(id, result)).pipe(Effect.interruptible, Effect.timeoutOrElse({
                         duration: limits.hook,
-                        orElse: () => Effect.fail(new ReactorError("Timeout", "H3 result hook exceeded its deadline")),
+                        orElse: () => Effect.fail(ReactorError.fromCode("Timeout", "H3 result hook exceeded its deadline")),
                     })));
                     // Preserve the entire secondary Cause for deliberate inspection,
                     // including defects; never substitute it for primary remote evidence.
                     if (Exit.isFailure(hook))
                         emit({
                             _tag: "Diagnostic",
-                            error: new ReactorError("InvalidState", "H3 result hook failed", {
+                            error: ReactorError.fromCode("InvalidState", "H3 result hook failed", {
                                 operation: "H3 result hook",
                                 detail: hook.cause,
                             }),
@@ -421,7 +551,7 @@ const build = (session, options) => Effect.gen(function* () {
                         : Effect.fail(result.failure);
                 })));
         },
-    }).pipe(Scope.provide(scope));
+    }).pipe(Scope.provide(executions));
     const prepare = (input, hooks = {}) => Effect.gen(function* () {
         yield* active("enqueue", true);
         const request = yield* pure(() => captureRequest(input, limits.prompt));
@@ -432,9 +562,9 @@ const build = (session, options) => Effect.gen(function* () {
     const prepareFrom = (preparation, hooks = {}) => Effect.gen(function* () {
         yield* active("enqueue", true);
         if (!Effect.isEffect(preparation))
-            return yield* Effect.fail(new ReactorError("InvalidInput", "H3 preparation must be an Effect", {
+            return yield* ReactorError.fromCode("InvalidInput", "H3 preparation must be an Effect", {
                 outcome: "not-submitted",
-            }));
+            });
         const id = yield* nextId();
         return yield* prepared(id, preparation.pipe(Effect.flatMap((input) => pure(() => {
             const request = captureRequest(input, limits.prompt);
@@ -443,14 +573,15 @@ const build = (session, options) => Effect.gen(function* () {
     });
     const contract = yield* session.schema.pipe(Effect.flatMap(({ openapi }) => pure(() => validateDeployment(openapi))), Effect.timeoutOrElse({
         duration: limits.setup,
-        orElse: () => Effect.fail(new ReactorError("Timeout", "H3 schema read timed out")),
+        orElse: () => Effect.fail(ReactorError.fromCode("Timeout", "H3 schema read timed out")),
     }));
     yield* refresh.pipe(Effect.timeoutOrElse({
         duration: limits.setup,
-        orElse: () => Effect.fail(new ReactorError("Timeout", "H3 initial state and queue reads timed out")),
+        orElse: () => Effect.fail(ReactorError.fromCode("Timeout", "H3 initial state and queue reads timed out")),
     }));
     if (state.snapshot()._tag !== "Ready")
-        return yield* Effect.fail(fatalError ?? new ReactorError("InvalidState", "H3 initial observations are unavailable"));
+        return yield* (fatalError ??
+            ReactorError.fromCode("InvalidState", "H3 initial observations are unavailable"));
     const provider = {
         sessionId: session.id,
         contract,
@@ -458,44 +589,47 @@ const build = (session, options) => Effect.gen(function* () {
         observe: (bounds) => Effect.gen(function* () {
             const stream = yield* events.subscribe(bounds);
             const initial = state.snapshot();
-            return { initial, revision: initial.revision, events: stream };
+            // The reducer may apply a queued event between the subscription and
+            // this read; the snapshot then covers it, so it is not repeated.
+            return {
+                initial,
+                revision: initial.revision,
+                events: Stream.filter(stream, (event) => !covered(event, initial.revision)),
+            };
         }),
         events: (bounds) => events.stream(bounds),
         failure: Deferred.await(fatal),
         acceptances: Effect.sync(() => Object.freeze([...acceptances.values()])),
         acceptance: (id) => Effect.sync(() => acceptances.get(id)),
+        operation: (submission) => operations.attach(submission.id),
         prepare,
         prepareFrom,
-        enqueue: (request) => prepare(request).pipe(Effect.mapError((error) => localFailure("enqueue", error)), Effect.flatMap((submission) => submission.submit)),
+        enqueue: (request) => prepare(request).pipe(Effect.mapError((error) => ReactorError.is(error) ? localFailure("enqueue", error) : error), Effect.flatMap((submission) => submission.submit)),
         getState,
         getQueue,
         refresh,
-        pop: (id) => checked("pop", () => clipId(id)).pipe(Effect.flatMap((id) => named("pop", { clip_id: id })), Effect.flatMap((reply) => reply.value.clip.clip_id === id
-            ? Effect.succeed(reply)
-            : Effect.fail(uncertain("pop", reply.source, "H3 pop returned a different clip")))),
+        pop: (id) => checked("pop", () => clipId(id)).pipe(Effect.flatMap((id) => named("pop", { clip_id: id })), Effect.filterOrFail((reply) => reply.value.clip.clip_id === id, (reply) => uncertain("pop", reply.source, "H3 pop returned a different clip"))),
         move: (id, position) => checked("move", () => ({
             clip_id: clipId(id),
             position: nonnegative(position, "position"),
-        })).pipe(Effect.flatMap((args) => named("move", args)), Effect.flatMap((reply) => reply.value.clip.clip_id === id
-            ? Effect.succeed(reply)
-            : Effect.fail(uncertain("move", reply.source, "H3 move returned a different clip")))),
+        })).pipe(Effect.flatMap((args) => named("move", args)), Effect.filterOrFail((reply) => reply.value.clip.clip_id === id, (reply) => uncertain("move", reply.source, "H3 move returned a different clip"))),
         play: (id) => checked("play", () => ({ clip_id: id === undefined ? "" : clipId(id) })).pipe(Effect.flatMap((args) => control("play", args))),
         stop: control("stop", {}),
         setSeed: (value) => checked("set_seed", () => nonnegative(value, "seed")).pipe(Effect.flatMap((seed) => named("set_seed", { seed }))),
         setClipSeconds: (value) => checked("set_clip_seconds", () => seconds(value)).pipe(Effect.flatMap((seconds) => named("set_clip_seconds", { seconds }))),
         setCanvas: (aspect) => checked("set_canvas", () => {
             if (!Object.hasOwn(canvases, aspect))
-                throw new ReactorError("InvalidInput", "Unsupported H3 canvas aspect");
+                throw ReactorError.fromCode("InvalidInput", "Unsupported H3 canvas aspect");
             return { aspect };
         }).pipe(Effect.flatMap((args) => named("set_canvas", args))),
         setAutoplay: (enabled) => checked("set_autoplay", () => {
             if (typeof enabled !== "boolean")
-                throw new ReactorError("InvalidInput", "Autoplay must be boolean");
+                throw ReactorError.fromCode("InvalidInput", "Autoplay must be boolean");
             return { enabled };
         }).pipe(Effect.flatMap((args) => named("set_autoplay", args))),
         setFlushOnClipEnd: (enabled) => checked("set_flush_on_clip_end", () => {
             if (typeof enabled !== "boolean")
-                throw new ReactorError("InvalidInput", "Flush setting must be boolean");
+                throw ReactorError.fromCode("InvalidInput", "Flush setting must be boolean");
             return { enabled };
         }).pipe(Effect.flatMap((args) => named("set_flush_on_clip_end", args))),
         reset: named("reset", {}),