the-moby-effect 1.51.7 → 1.54.2

package/src/internal/demux/multiplexed.ts

@@ -1,13 +1,12 @@
-import type * as HttpClientResponse from "@effect/platform/HttpClientResponse";
-import type * as ParseResult from "effect/ParseResult";
+import type * as Array from "effect/Array";
 import type * as Scope from "effect/Scope";
-import type * as MobyDemux from "../../MobyDemux.js";
+import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse";
 
-import * as Socket from "@effect/platform/Socket";
 import * as Channel from "effect/Channel";
 import * as Chunk from "effect/Chunk";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
+import * as Filter from "effect/Filter";
 import * as Function from "effect/Function";
 import * as Pipeable from "effect/Pipeable";
 import * as Predicate from "effect/Predicate";
@@ -16,6 +15,10 @@ import * as Schema from "effect/Schema";
 import * as Sink from "effect/Sink";
 import * as Stream from "effect/Stream";
 import * as Tuple from "effect/Tuple";
+import * as Socket from "effect/unstable/socket/Socket";
+
+import type * as MobyDemux from "../../MobyDemux.js";
+
 import * as internalCompressed from "./compressed.js";
 
 /** @internal */
@@ -44,11 +47,11 @@ export const makeMultiplexedSocket = (underlying: Socket.Socket): MobyDemux.Mult
 /** @internal */
 export const makeMultiplexedChannel = <IE = unknown, OE = Socket.SocketError, R = never>(
     underlying: Channel.Channel<
-        Chunk.Chunk<Uint8Array>,
-        Chunk.Chunk<string | Uint8Array | Socket.CloseEvent>,
+        Array.NonEmptyReadonlyArray<Uint8Array>,
         OE,
-        IE,
         void,
+        Array.NonEmptyReadonlyArray<string | Uint8Array | Socket.CloseEvent>,
+        IE,
         unknown,
         R
     >
@@ -104,15 +107,7 @@ export type MultiplexedAccumulator = {
 };
 
 /** @internal */
-export interface $MultiplexedSchema extends Schema.Tuple<
-    [Schema.Enums<typeof MultiplexedHeaderType>, Schema.Schema<Uint8Array, ReadonlyArray<number>, never>]
-> {}
-
-/** @internal */
-export const MultiplexedSchema: $MultiplexedSchema = Schema.Tuple(
-    Schema.Enums(MultiplexedHeaderType),
-    Schema.Uint8Array
-);
+export const MultiplexedSchema = Schema.Tuple([Schema.Enum(MultiplexedHeaderType), Schema.Uint8Array]);
 
 /** @internal */
 export const asMultiplexedChannel = <IE = never, OE = Socket.SocketError, R = never>(
@@ -130,23 +125,19 @@ export const asMultiplexedChannel = <IE = never, OE = Socket.SocketError, R = ne
 export const multiplexedToStream = <IE = never, OE = Socket.SocketError, R = never>(
     input: MobyDemux.EitherMultiplexedInput<IE, OE, R>
 ): Stream.Stream<Uint8Array, IE | OE, R> =>
-    Channel.toStream(
-        asMultiplexedChannel(input).underlying as Channel.Channel<
-            Chunk.Chunk<Uint8Array>,
-            unknown,
-            IE | OE,
-            unknown,
-            void,
-            unknown,
-            R
-        >
-    );
+    Stream.pipeThroughChannelOrFail(Stream.empty, asMultiplexedChannel(input).underlying);
 
 /** @internal */
 export const multiplexedToSink = <IE = never, OE = Socket.SocketError, R = never>(
     input: MobyDemux.EitherMultiplexedInput<IE, OE, R>
 ): Sink.Sink<void, string | Uint8Array | Socket.CloseEvent, Uint8Array, IE | OE, R> =>
-    Channel.toSink(asMultiplexedChannel(input).underlying);
+    Sink.fromChannel(
+        Function.pipe(
+            asMultiplexedChannel(input).underlying,
+            Channel.drain,
+            Channel.mapDone(() => [void 0] as const)
+        )
+    );
 
 /** @internal */
 export const multiplexedFromStreamWith =
@@ -159,21 +150,21 @@ export const multiplexedFromStream = <E, R>(
     input: Stream.Stream<Uint8Array, E, R>
 ): MobyDemux.MultiplexedChannel<never, E, R> => multiplexedFromStreamWith<never>()(input);
 
-/** @internal */
-export const multiplexedFromSink = <E, R>(
-    input: Sink.Sink<void, string | Uint8Array | Socket.CloseEvent, Uint8Array, E, R>
-): MobyDemux.MultiplexedChannel<never, E, R> => makeMultiplexedChannel<never, E, R>(Sink.toChannel(input));
+// /** @internal */
+// export const multiplexedFromSink = <E, R>(
+//     input: Sink.Sink<void, string | Uint8Array | Socket.CloseEvent, Uint8Array, E, R>
+// ): MobyDemux.MultiplexedChannel<never, E, R> => makeMultiplexedChannel<never, E, R>(Sink.toChannel(input));
 
 /** @internal */
-export const demuxMultiplexedFolderSink: Sink.Sink<MultiplexedAccumulator, number, number, never, never> = Sink.fold(
-    {
+export const demuxMultiplexedFolderSink: Sink.Sink<MultiplexedAccumulator, number, number> = Sink.reduceWhile(
+    (): MultiplexedAccumulator => ({
         headerBytesRead: 0,
         messageBytesRead: 0,
         headerBuffer: Chunk.empty<number>(),
         messageBuffer: Chunk.empty<number>(),
-        messageSize: undefined as number | undefined,
-        messageType: undefined as number | undefined,
-    },
+        messageSize: undefined,
+        messageType: undefined,
+    }),
     ({ messageBytesRead, messageSize }) => messageSize === undefined || messageBytesRead < messageSize,
     (accumulator, input: number) => {
         // If we have not read the entire header yet
@@ -210,13 +201,15 @@ export const demuxMultiplexedFolderSink: Sink.Sink<MultiplexedAccumulator, numbe
 /** @internal */
 export const aggregate = <E, R>(
     multiplexedStream: Stream.Stream<Uint8Array, E, R>
-): Stream.Stream<readonly [MultiplexedHeaderType, Uint8Array], ParseResult.ParseError | E, R> =>
+): Stream.Stream<readonly [MultiplexedHeaderType, Uint8Array], Schema.SchemaError | E, R> =>
     Function.pipe(
         multiplexedStream,
-        Stream.mapConcat(Function.identity),
+        Stream.flattenIterable,
         Stream.aggregateWithin(demuxMultiplexedFolderSink, Schedule.fixed(Duration.infinity)),
-        Stream.map(({ messageBuffer, messageType }) => Tuple.make(messageType, Chunk.toReadonlyArray(messageBuffer))),
-        Stream.flatMap(Schema.decodeUnknown(MultiplexedSchema))
+        Stream.map(({ messageBuffer, messageType }) =>
+            Tuple.make(messageType, new Uint8Array(Chunk.toReadonlyArray(messageBuffer)))
+        ),
+        Stream.mapEffect((multiplexed) => Schema.decodeUnknownEffect(MultiplexedSchema)(multiplexed))
     );
 
 /** @internal */
@@ -228,28 +221,22 @@ export const demuxMultiplexedToSingleSink = Function.dual<
         options?: { encoding?: string | undefined } | undefined
     ) => <IE = never, OE = Socket.SocketError, R3 = never>(
         socket: MobyDemux.EitherMultiplexedInput<E1 | IE, OE, R3>
-    ) => Effect.Effect<
-        A1,
-        E1 | E2 | IE | OE | ParseResult.ParseError,
-        Exclude<R1, Scope.Scope> | Exclude<R2, Scope.Scope> | Exclude<R3, Scope.Scope>
-    >,
+    ) => Effect.Effect<A1, E1 | E2 | IE | OE | Schema.SchemaError, R1 | R2 | R3>,
     // One sink, data-first signature.
     <A1, L1, E1, E2, R1, R2, IE = never, OE = Socket.SocketError, R3 = never>(
         socket: MobyDemux.EitherMultiplexedInput<E1 | IE, OE, R3>,
         source: Stream.Stream<string | Uint8Array | Socket.CloseEvent, E1, R1>,
         sink: Sink.Sink<A1, readonly [MultiplexedHeaderType, string], L1, E2, R2>,
         options?: { encoding?: string | undefined } | undefined
-    ) => Effect.Effect<
-        A1,
-        E1 | E2 | IE | OE | ParseResult.ParseError,
-        Exclude<R1, Scope.Scope> | Exclude<R2, Scope.Scope> | Exclude<R3, Scope.Scope>
-    >
+    ) => Effect.Effect<A1, E1 | E2 | IE | OE | Schema.SchemaError, R1 | R2 | R3>
 >(
     (arguments_) => isMultiplexedSocket(arguments_[0]) || isMultiplexedChannel(arguments_[0]),
     (socket, source, sink, options) => {
         const { underlying } = asMultiplexedChannel(socket);
         const textDecoder = new TextDecoder(options?.encoding);
-        const toStrings = Tuple.mapSecond((bytes: Uint8Array) => textDecoder.decode(bytes, { stream: true }));
+        const toStrings = ([messageType, bytes]: readonly [MultiplexedHeaderType, Uint8Array]) =>
+            Tuple.make(messageType, textDecoder.decode(bytes, { stream: true }));
+
         return Function.pipe(
             source,
             Stream.pipeThroughChannelOrFail(underlying),
@@ -272,8 +259,8 @@ export const demuxMultiplexedToSeparateSinks = Function.dual<
         socket: MobyDemux.EitherMultiplexedInput<E1 | IE, OE, R4>
     ) => Effect.Effect<
         MobyDemux.CompressedDemuxOutput<A1, A2>,
-        E1 | E2 | E3 | IE | OE | ParseResult.ParseError,
-        Exclude<R1, Scope.Scope> | Exclude<R2, Scope.Scope> | Exclude<R3, Scope.Scope> | Exclude<R4, Scope.Scope>
+        E1 | E2 | E3 | IE | OE | Schema.SchemaError,
+        Exclude<R1 | R2 | R3 | R4, Scope.Scope>
     >,
     // Two sinks, data-first signature.
     <A1, A2, L1, L2, E1, E2, E3, R1, R2, R3, IE = never, OE = Socket.SocketError, R4 = never>(
@@ -284,8 +271,8 @@ export const demuxMultiplexedToSeparateSinks = Function.dual<
         options?: { bufferSize?: number | undefined; encoding?: string | undefined } | undefined
     ) => Effect.Effect<
         MobyDemux.CompressedDemuxOutput<A1, A2>,
-        E1 | E2 | E3 | IE | OE | ParseResult.ParseError,
-        Exclude<R1, Scope.Scope> | Exclude<R2, Scope.Scope> | Exclude<R3, Scope.Scope> | Exclude<R4, Scope.Scope>
+        E1 | E2 | E3 | IE | OE | Schema.SchemaError,
+        Exclude<R1 | R2 | R3 | R4, Scope.Scope>
     >
 >(
     (arguments_) => isMultiplexedSocket(arguments_[0]) || isMultiplexedChannel(arguments_[0]),
@@ -294,25 +281,31 @@ export const demuxMultiplexedToSeparateSinks = Function.dual<
             source,
             Stream.pipeThroughChannelOrFail(asMultiplexedChannel(socket).underlying),
             aggregate,
-            Stream.partition(([messageType]) => messageType === MultiplexedHeaderType.Stderr, {
-                bufferSize: options?.bufferSize,
-            }),
+            Stream.partition(
+                Filter.fromPredicate(
+                    ([messageType]: Schema.Schema.Type<typeof MultiplexedSchema>) =>
+                        messageType === MultiplexedHeaderType.Stderr
+                ),
+                { bufferSize: options?.bufferSize ?? 16 }
+            ),
             Effect.map(
-                Tuple.mapBoth({
-                    onFirst: Function.flow(
-                        Stream.map(Tuple.getSecond),
-                        Stream.decodeText(options?.encoding),
-                        Stream.run(sink1)
-                    ),
-                    onSecond: Function.flow(
-                        Stream.map(Tuple.getSecond),
-                        Stream.decodeText(options?.encoding),
-                        Stream.run(sink2)
-                    ),
-                })
+                ([stdoutStream, stderrStream]) =>
+                    [
+                        Function.pipe(
+                            stdoutStream,
+                            Stream.map(Tuple.get(1)),
+                            Stream.decodeText({ encoding: options?.encoding }),
+                            Stream.run(sink1)
+                        ),
+                        Function.pipe(
+                            stderrStream,
+                            Stream.map(Tuple.get(1)),
+                            Stream.decodeText({ encoding: options?.encoding }),
+                            Stream.run(sink2)
+                        ),
+                    ] as const
             ),
-            Effect.map(Effect.allWith({ concurrency: 2 })),
-            Effect.flatten,
+            Effect.flatMap((streamers) => Effect.all(streamers, { concurrency: 2 })),
             Effect.map(internalCompressed.compressDemuxOutput),
             Effect.scoped
         )

package/src/internal/demux/pack.ts

@@ -1,24 +1,33 @@
+import type * as Cause from "effect/Cause";
 import type * as Scope from "effect/Scope";
-import type * as MobyDemux from "../../MobyDemux.js";
 
-import * as Socket from "@effect/platform/Socket";
+import * as Array from "effect/Array";
 import * as Channel from "effect/Channel";
-import * as Chunk from "effect/Chunk";
 import * as Effect from "effect/Effect";
-import * as Either from "effect/Either";
-import * as Exit from "effect/Exit";
+import * as Fiber from "effect/Fiber";
 import * as Function from "effect/Function";
 import * as Match from "effect/Match";
-import * as Option from "effect/Option";
 import * as Predicate from "effect/Predicate";
+import * as Pull from "effect/Pull";
 import * as Queue from "effect/Queue";
 import * as Sink from "effect/Sink";
 import * as Stream from "effect/Stream";
+import * as Socket from "effect/unstable/socket/Socket";
+
+import type * as MobyDemux from "../../MobyDemux.js";
 
-import { makeMultiplexedChannel, multiplexedFromStreamWith, MultiplexedHeaderType } from "./multiplexed.js";
+import { makeMultiplexedChannel, MultiplexedHeaderType } from "./multiplexed.js";
 import { demuxStdioRawToSeparateSinks } from "./raw.js";
 
-/** @internal */
+/**
+ * The pack demux driver runs exactly once, forked into the caller's scope,
+ * and the returned multiplexed channel is pure queue plumbing. This keeps the
+ * demux deterministic - the raw stdio inputs start being consumed as soon as
+ * pack is called, and there is no racing over which consumer starts the
+ * driver.
+ *
+ * @internal
+ */
 export const pack = Function.dual<
     <
         IE1 = never,
@@ -42,9 +51,9 @@ export const pack = Function.dual<
     }) => (
         stdio: MobyDemux.HeterogeneousStdioRawInput<IE1, IE2, IE3, OE1, OE2, OE3, R1, R2, R3>
     ) => Effect.Effect<
-        MobyDemux.MultiplexedChannel<IE1 | IE2 | IE3, IE1 | IE2 | IE3 | OE1 | OE2 | OE3, never>,
+        MobyDemux.MultiplexedChannel<IE1, IE1 | IE2 | IE3 | OE1 | OE2 | OE3, never>,
         never,
-        Exclude<R1, Scope.Scope> | Exclude<R2, Scope.Scope> | Exclude<R3, Scope.Scope>
+        R1 | R2 | R3 | Scope.Scope
     >,
     <
         IE1 = never,
@@ -69,9 +78,9 @@ export const pack = Function.dual<
             readonly encoding?: string | undefined;
         }
     ) => Effect.Effect<
-        MobyDemux.MultiplexedChannel<IE1 | IE2 | IE3, IE1 | IE2 | IE3 | OE1 | OE2 | OE3, never>,
+        MobyDemux.MultiplexedChannel<IE1, IE1 | IE2 | IE3 | OE1 | OE2 | OE3, never>,
         never,
-        Exclude<R1, Scope.Scope> | Exclude<R2, Scope.Scope> | Exclude<R3, Scope.Scope>
+        R1 | R2 | R3 | Scope.Scope
     >
 >(
     (arguments_) => "stdin" in arguments_[0] || "stdout" in arguments_[0] || "stderr" in arguments_[0],
@@ -98,56 +107,43 @@ export const pack = Function.dual<
             readonly encoding?: string | undefined;
         }
     ) {
-        const mutex = yield* Effect.makeSemaphore(1);
-        const context = yield* Effect.context<
-            Exclude<R1, Scope.Scope> | Exclude<R2, Scope.Scope> | Exclude<R3, Scope.Scope>
-        >();
+        type CanReceive = Uint8Array | string | Socket.CloseEvent;
+        type PackError = IE1 | IE2 | IE3 | OE1 | OE2 | OE3;
 
+        // Internal buffers. The consumer queues carry the driver's failure so
+        // that a demux error surfaces on the returned multiplexed channel.
         const capacity = options.requestedCapacity;
-        type CanReceive = Uint8Array | string | Socket.CloseEvent;
-        const stdoutConsumerQueue = yield* Queue.bounded<string>(
+        const stdoutConsumerQueue = yield* Queue.bounded<string, PackError | Cause.Done>(
             typeof capacity === "number" ? capacity : capacity.stdoutCapacity
         );
-        const stderrConsumerQueue = yield* Queue.bounded<string>(
+        const stderrConsumerQueue = yield* Queue.bounded<string, PackError | Cause.Done>(
             typeof capacity === "number" ? capacity : capacity.stderrCapacity
         );
-        const stdinProducerQueue = yield* Queue.bounded<Either.Either<Chunk.Chunk<CanReceive>, Exit.Exit<void, IE1>>>(
+        const stdinProducerQueue = yield* Queue.bounded<Uint8Array, IE1 | Cause.Done>(
             typeof capacity === "number" ? capacity : capacity.stdinCapacity
         );
 
-        // Demux everything to and fro the correct places. We can touch
-        // this more than once because it is wrapped in the mutex.
-        const mother = demuxStdioRawToSeparateSinks(
+        // The driver demuxes the raw stdio inputs into the consumer queues
+        // and pumps the stdin producer queue into them. On completion the
+        // consumer queues are ended; on failure the failure is forwarded to
+        // them so it surfaces on the returned multiplexed channel.
+        const driver = yield* demuxStdioRawToSeparateSinks(
             stdio,
             {
-                stdin: Function.pipe(
-                    Stream.fromQueue(stdinProducerQueue, { shutdown: true }),
-                    Stream.map(
-                        Either.match({
-                            onRight: Exit.succeed,
-                            onLeft: Function.flow(
-                                Exit.mapBoth({
-                                    onFailure: Option.some,
-                                    onSuccess: Function.compose(Option.none, Exit.fail),
-                                }),
-                                Exit.flatten
-                            ),
-                        })
-                    ),
-                    Stream.flattenExitOption,
-                    Stream.flattenChunks
-                ),
-                stdout: Sink.fromQueue(stdoutConsumerQueue, { shutdown: true }),
-                stderr: Sink.fromQueue(stderrConsumerQueue, { shutdown: true }),
+                stdin: Stream.fromQueue(stdinProducerQueue),
+                stdout: Sink.forEachArray((chunk) => Queue.offerAll(stdoutConsumerQueue, chunk)),
+                stderr: Sink.forEachArray((chunk) => Queue.offerAll(stderrConsumerQueue, chunk)),
             },
             {
                 encoding: options?.encoding,
             }
-        )
-            .pipe(mutex.withPermitsIfAvailable(1))
-            .pipe(Effect.asVoid)
-            .pipe(Channel.fromEffect)
-            .pipe(Channel.provideContext(context));
+        ).pipe(
+            Effect.tap(() => Effect.andThen(Queue.end(stdoutConsumerQueue), Queue.end(stderrConsumerQueue))),
+            Effect.tapCause((cause) =>
+                Effect.andThen(Queue.failCause(stdoutConsumerQueue, cause), Queue.failCause(stderrConsumerQueue, cause))
+            ),
+            Effect.forkScoped
+        );
 
         // Convert the streams to the multiplexed streams
         const textEncoder = new TextEncoder();
@@ -181,24 +177,40 @@ export const pack = Function.dual<
                 return result;
             };
 
-        const concurrency = { concurrent: true } as const;
-        const independentStdinChannel = Channel.toQueue(stdinProducerQueue)
-            .pipe(Channel.mapInput(Function.constVoid))
-            .pipe(Channel.mapInputError(Function.unsafeCoerce<unknown, IE1 | IE2 | IE3>))
-            .pipe(Channel.mapInputIn((input: Chunk.Chunk<CanReceive>) => Chunk.map(input, encode)))
-            .pipe(Channel.zipLeft(mother, concurrency));
+        // Forwards this channel's input into the stdin producer queue,
+        // propagating input errors and the end-of-input signal.
+        const feedStdin = (
+            upstream: Pull.Pull<Array.NonEmptyReadonlyArray<CanReceive>, IE1, unknown, never>
+        ): Effect.Effect<void> =>
+            upstream.pipe(
+                Effect.flatMap((chunk) => Queue.offerAll(stdinProducerQueue, Array.map(chunk, encode))),
+                Effect.forever,
+                Pull.catchDone(() => Queue.end(stdinProducerQueue)),
+                Effect.catchCause((cause) => Queue.failCause(stdinProducerQueue, cause)),
+                Effect.asVoid
+            );
 
-        const { underlying: independentStdoutChannel } = Stream.fromQueue(stdoutConsumerQueue)
-            .pipe(Stream.encodeText)
-            .pipe(Stream.map(mapOutEntry(MultiplexedHeaderType.Stdout)))
-            .pipe(multiplexedFromStreamWith<IE1 | IE2 | IE3>());
+        // The stdin side completes (or fails) with the driver.
+        const independentStdinChannel = Channel.fromEffectDrain(Fiber.join(driver)).pipe(
+            Channel.embedInput<Array.NonEmptyReadonlyArray<CanReceive>, IE1, unknown, never>(feedStdin)
+        );
 
-        const { underlying: independentStderrChannel } = Stream.fromQueue(stderrConsumerQueue)
-            .pipe(Stream.encodeText)
-            .pipe(Stream.map(mapOutEntry(MultiplexedHeaderType.Stderr)))
-            .pipe(multiplexedFromStreamWith<IE1 | IE2 | IE3>());
+        const independentStdoutChannel = Stream.fromQueue(stdoutConsumerQueue).pipe(
+            Stream.encodeText,
+            Stream.map(mapOutEntry(MultiplexedHeaderType.Stdout)),
+            Stream.toChannel
+        );
 
-        const mixedOutputChannel = Channel.zip(independentStdoutChannel, independentStderrChannel, concurrency);
-        return makeMultiplexedChannel(Channel.zip(independentStdinChannel, mixedOutputChannel, concurrency));
+        const independentStderrChannel = Stream.fromQueue(stderrConsumerQueue).pipe(
+            Stream.encodeText,
+            Stream.map(mapOutEntry(MultiplexedHeaderType.Stderr)),
+            Stream.toChannel
+        );
+
+        const haltStrategy = { haltStrategy: "both" } as const;
+        const mixedOutputChannel = Channel.merge(independentStdoutChannel, independentStderrChannel, haltStrategy);
+        return makeMultiplexedChannel<IE1, IE1 | IE2 | IE3 | OE1 | OE2 | OE3, never>(
+            Channel.merge(independentStdinChannel, mixedOutputChannel, haltStrategy)
+        );
     })
 );