starpc 0.18.3 → 0.19.1

package/dist/rpcstream/pushable-sink.js

@@ -0,0 +1 @@
+"use strict";

package/dist/rpcstream/rpcstream.js

@@ -31,7 +31,7 @@ export async function openRpcStream(componentId, caller, waitAck) {
         }
     }
     // build & return the data stream
-    return new RpcStream(packetSink, packetIt); // packetSource)
+    return new RpcStream(packetSink, packetIt);
 }
 // buildRpcStreamOpenStream builds a OpenStream func with a RpcStream.
 export function buildRpcStreamOpenStream(componentId, caller) {

package/dist/srpc/client.js

@@ -3,8 +3,7 @@ import { pushable } from 'it-pushable';
 import { ERR_RPC_ABORT } from './errors.js';
 import { ClientRPC } from './client-rpc.js';
 import { writeToPushable } from './pushable.js';
-import { decodePacketSource, encodePacketSource, parseLengthPrefixTransform, prependLengthPrefixTransform, } from './packet.js';
-import { combineUint8ArrayListTransform } from './array-list.js';
+import { decodePacketSource, encodePacketSource, } from './packet.js';
 import { OpenStreamCtr } from './open-stream-ctr.js';
 // Client implements the ts-proto Rpc interface with the drpcproto protocol.
 export class Client {
@@ -75,12 +74,12 @@ export class Client {
             throw new Error(ERR_RPC_ABORT);
         }
         const openStreamFn = await this.openStreamCtr.wait();
-        const conn = await openStreamFn();
+        const stream = await openStreamFn();
         const call = new ClientRPC(rpcService, rpcMethod);
         abortSignal?.addEventListener('abort', () => {
             call.close(new Error(ERR_RPC_ABORT));
         });
-        pipe(conn, parseLengthPrefixTransform(), combineUint8ArrayListTransform(), decodePacketSource, call, encodePacketSource, prependLengthPrefixTransform(), conn);
+        pipe(stream, decodePacketSource, call, encodePacketSource, stream);
         await call.writeCallStart(data || undefined);
         return call;
     }

package/dist/srpc/common-rpc.js

@@ -77,7 +77,7 @@ export class CommonRPC {
                 asError = new Error('error handling packet');
             }
             this.close(asError);
-            throw asError;
+            // throw asError
         }
     }
     // handleCallStart handles a CallStart packet.

package/dist/srpc/conn.d.ts

@@ -9,7 +9,7 @@ export interface ConnParams {
     direction?: Direction;
 }
 export interface StreamHandler {
-    handleStream(strm: SRPCStream): void;
+    handlePacketStream(strm: SRPCStream): void;
 }
 export declare function streamToSRPCStream(stream: Duplex<Uint8ArrayList, Uint8ArrayList | Uint8Array>): SRPCStream;
 export declare class Conn implements Duplex<Uint8Array> {

package/dist/srpc/conn.js

@@ -1,13 +1,19 @@
 import { pipe } from 'it-pipe';
 import { yamux } from '@chainsafe/libp2p-yamux';
 import isPromise from 'is-promise';
+import { pushable } from 'it-pushable';
 import { Client } from './client.js';
 import { combineUint8ArrayListTransform } from './array-list.js';
+import { parseLengthPrefixTransform, prependLengthPrefixTransform } from './packet.js';
+import { buildPushableSink } from './pushable.js';
 // streamToSRPCStream converts a Stream to a SRPCStream.
+// uses length-prefix for packet framing
 export function streamToSRPCStream(stream) {
+    const pushSink = pushable({ objectMode: true });
+    pipe(pushSink, prependLengthPrefixTransform(), stream.sink);
     return {
-        source: pipe(stream, combineUint8ArrayListTransform()),
-        sink: stream.sink,
+        source: pipe(stream, parseLengthPrefixTransform(), combineUint8ArrayListTransform()),
+        sink: buildPushableSink(pushSink),
     };
 }
 // Conn implements a generic connection with a two-way stream.
@@ -46,9 +52,13 @@ export class Conn {
     }
     // openStream implements the client open stream function.
     async openStream() {
-        const stream = this.muxer.newStream();
-        if (isPromise(stream)) {
-            return streamToSRPCStream(await stream);
+        const streamPromise = this.muxer.newStream();
+        let stream;
+        if (isPromise(streamPromise)) {
+            stream = await streamPromise;
+        }
+        else {
+            stream = streamPromise;
         }
         return streamToSRPCStream(stream);
     }
@@ -62,6 +72,6 @@ export class Conn {
         if (!server) {
             return strm.abort(new Error('server not implemented'));
         }
-        server.handleStream(streamToSRPCStream(strm));
+        server.handlePacketStream(streamToSRPCStream(strm));
     }
 }

package/dist/srpc/errors.d.ts

@@ -1,2 +1,3 @@
 export declare const ERR_RPC_ABORT = "ERR_RPC_ABORT";
 export declare function isAbortError(err: unknown): boolean;
+export declare function castToError(err: any, defaultMsg?: string): Error;

package/dist/srpc/errors.js

@@ -8,3 +8,26 @@ export function isAbortError(err) {
     const message = err.message;
     return message === ERR_RPC_ABORT;
 }
+// castToError casts an object to an Error.
+// if err is a string, uses it as the message.
+// if err is undefined, returns new Error(defaultMsg)
+export function castToError(err, defaultMsg) {
+    defaultMsg = defaultMsg || 'error';
+    if (!err) {
+        return new Error(defaultMsg);
+    }
+    if (typeof err === 'string') {
+        return new Error(err);
+    }
+    const asError = err;
+    if (asError.message) {
+        return asError;
+    }
+    if (err.toString) {
+        const errString = err.toString();
+        if (errString) {
+            return new Error(errString);
+        }
+    }
+    return new Error(defaultMsg);
+}

package/dist/srpc/index.d.ts

@@ -1,5 +1,5 @@
 export type { PacketHandler, Stream, OpenStreamFunc } from './stream.js';
-export { ERR_RPC_ABORT, isAbortError } from './errors.js';
+export { ERR_RPC_ABORT, isAbortError, castToError } from './errors.js';
 export { Client } from './client.js';
 export { Server } from './server.js';
 export { Conn, ConnParams } from './conn.js';
@@ -9,6 +9,8 @@ export { Mux, StaticMux, createMux } from './mux.js';
 export { BroadcastChannelDuplex, newBroadcastChannelDuplex, BroadcastChannelConn, } from './broadcast-channel.js';
 export { MessagePortIterable, newMessagePortIterable, MessagePortConn, } from './message-port.js';
 export { MessageDefinition, DecodeMessageTransform, buildDecodeMessageTransform, EncodeMessageTransform, buildEncodeMessageTransform, memoProto, memoProtoDecode, } from './message.js';
+export { parseLengthPrefixTransform, prependLengthPrefixTransform, decodePacketSource, encodePacketSource, } from './packet.js';
+export { combineUint8ArrayListTransform } from './array-list.js';
 export { ValueCtr } from './value-ctr.js';
 export { OpenStreamCtr } from './open-stream-ctr.js';
-export { writeToPushable } from './pushable';
+export { writeToPushable, buildPushableSink } from './pushable.js';

package/dist/srpc/index.js

@@ -1,4 +1,4 @@
-export { ERR_RPC_ABORT, isAbortError } from './errors.js';
+export { ERR_RPC_ABORT, isAbortError, castToError } from './errors.js';
 export { Client } from './client.js';
 export { Server } from './server.js';
 export { Conn } from './conn.js';
@@ -8,6 +8,8 @@ export { StaticMux, createMux } from './mux.js';
 export { BroadcastChannelDuplex, newBroadcastChannelDuplex, BroadcastChannelConn, } from './broadcast-channel.js';
 export { MessagePortIterable, newMessagePortIterable, MessagePortConn, } from './message-port.js';
 export { buildDecodeMessageTransform, buildEncodeMessageTransform, memoProto, memoProtoDecode, } from './message.js';
+export { parseLengthPrefixTransform, prependLengthPrefixTransform, decodePacketSource, encodePacketSource, } from './packet.js';
+export { combineUint8ArrayListTransform } from './array-list.js';
 export { ValueCtr } from './value-ctr.js';
 export { OpenStreamCtr } from './open-stream-ctr.js';
-export { writeToPushable } from './pushable';
+export { writeToPushable, buildPushableSink } from './pushable.js';

package/dist/srpc/pushable.d.ts

@@ -1,2 +1,4 @@
 import { Pushable } from 'it-pushable';
+import { Sink } from 'it-stream-types';
 export declare function writeToPushable<T>(dataSource: AsyncIterable<T>, out: Pushable<T>): Promise<void>;
+export declare function buildPushableSink<T>(target: Pushable<T>): Sink<T>;

package/dist/srpc/pushable.js

@@ -1,3 +1,4 @@
+import { castToError } from './errors.js';
 // writeToPushable writes the incoming server data to the pushable.
 export async function writeToPushable(dataSource, out) {
     try {
@@ -11,3 +12,25 @@ export async function writeToPushable(dataSource, out) {
         throw err;
     }
 }
+// buildPushableSink builds a Sink from a Pushable.
+export function buildPushableSink(target) {
+    return async function pushableSink(source) {
+        try {
+            for await (const pkt of source) {
+                if (Array.isArray(pkt)) {
+                    for (const p of pkt) {
+                        target.push(p);
+                    }
+                }
+                else {
+                    target.push(pkt);
+                }
+            }
+            target.end();
+        }
+        catch (err) {
+            const error = castToError(err);
+            target.end(error);
+        }
+    };
+}

package/dist/srpc/server.d.ts

@@ -10,8 +10,7 @@ export declare class Server implements StreamHandler {
     constructor(lookupMethod: LookupMethod);
     get rpcStreamHandler(): RpcStreamHandler;
     startRpc(): ServerRPC;
-    handleStream(stream: Stream): ServerRPC;
-    handleDuplex(stream: Duplex<Uint8Array, Uint8Array>): ServerRPC;
-    handlePacketDuplex(stream: Duplex<Uint8Array>): ServerRPC;
-    handlePacketStream(stream: Duplex<Packet>): ServerRPC;
+    handleFragmentStream(stream: Stream): ServerRPC;
+    handlePacketStream(stream: Stream): ServerRPC;
+    handlePacketDuplex(stream: Duplex<Packet>): ServerRPC;
 }

package/dist/srpc/server.js

@@ -8,33 +8,32 @@ export class Server {
         this.lookupMethod = lookupMethod;
     }
     // rpcStreamHandler implements the RpcStreamHandler interface.
+    // uses handlePacketDuplex (expects 1 buf = 1 Packet)
     get rpcStreamHandler() {
-        return this.handleStream.bind(this);
+        return this.handlePacketStream.bind(this);
     }
     // startRpc starts a new server-side RPC.
     // the returned RPC handles incoming Packets.
     startRpc() {
         return new ServerRPC(this.lookupMethod);
     }
-    // handleStream handles an incoming Uint8Array message duplex.
-    handleStream(stream) {
-        return this.handleDuplex(stream);
-    }
-    // handleDuplex handles an incoming message duplex.
-    handleDuplex(stream) {
+    // handleFragmentStream handles an incoming stream.
+    // assumes that stream does not maintain packet framing.
+    // uses length-prefixed packets for packet framing.
+    handleFragmentStream(stream) {
         const rpc = this.startRpc();
         pipe(stream, parseLengthPrefixTransform(), combineUint8ArrayListTransform(), decodePacketSource, rpc, encodePacketSource, prependLengthPrefixTransform(), combineUint8ArrayListTransform(), stream);
         return rpc;
     }
-    // handlePacketDuplex handles an incoming Uint8Array duplex.
-    // skips the packet length prefix transform.
-    handlePacketDuplex(stream) {
+    // handlePacketStream handles an incoming Uint8Array duplex.
+    // the stream has one Uint8Array per packet w/o length prefix.
+    handlePacketStream(stream) {
         const rpc = this.startRpc();
         pipe(stream, decodePacketSource, rpc, encodePacketSource, stream);
         return rpc;
     }
-    // handlePacketStream handles an incoming Packet duplex.
-    handlePacketStream(stream) {
+    // handlePacketDuplex handles an incoming Packet duplex.
+    handlePacketDuplex(stream) {
         const rpc = this.startRpc();
         pipe(stream, rpc, stream);
         return rpc;

package/go.mod

@@ -1,6 +1,6 @@
 module github.com/aperturerobotics/starpc
 
-go 1.18
+go 1.19
 
 require (
 	github.com/pkg/errors v0.9.1 // latest
@@ -9,8 +9,8 @@ require (
 )
 
 require (
-	github.com/aperturerobotics/util v1.0.6-0.20230313071030-6260e05b62e5 // latest
-	github.com/libp2p/go-libp2p v0.26.3 // latest
+	github.com/aperturerobotics/util v1.1.1 // latest
+	github.com/libp2p/go-libp2p v0.26.1-0.20230405232726-15ec1494039f // latest
 	github.com/libp2p/go-yamux/v4 v4.0.1-0.20220919134236-1c09f2ab3ec1 // master
 	github.com/sirupsen/logrus v1.9.0 // latest
 )

package/go.sum

@@ -1,5 +1,5 @@
-github.com/aperturerobotics/util v1.0.6-0.20230313071030-6260e05b62e5 h1:SVQsecow4oFN4ra6SURikAJROdvSYY2lI0vujic5veU=
-github.com/aperturerobotics/util v1.0.6-0.20230313071030-6260e05b62e5/go.mod h1:45zfYCir2T5/KE+O9CyYdZPrsXZtOEAuA4UU79rRMN4=
+github.com/aperturerobotics/util v1.1.1 h1:lZye9i6tBN36mIki/V1Mk5r5JcegkiQBeaME23UULG0=
+github.com/aperturerobotics/util v1.1.1/go.mod h1:VB5a5FURpm7EujJsErIY8UZrhrLPN/bj84H8FxN0gD4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -48,8 +48,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
-github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM=
-github.com/libp2p/go-libp2p v0.26.3/go.mod h1:x75BN32YbwuY0Awm2Uix4d4KOz+/4piInkp4Wr3yOo8=
+github.com/libp2p/go-libp2p v0.26.1-0.20230405232726-15ec1494039f h1:lDBF6Sto3oiEXc+k9c57OAoxOqqJj7sXynw66TyRH+A=
+github.com/libp2p/go-libp2p v0.26.1-0.20230405232726-15ec1494039f/go.mod h1:HKQUKIQ5NhzabNMWq9Wczcs5Ksdx4LQ/ISAq4MRqBHQ=
 github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA=
 github.com/libp2p/go-yamux/v4 v4.0.1-0.20220919134236-1c09f2ab3ec1 h1:fxyHejZvFIBqlznQDuHUkw9BThYKyoT3DclP/C2g8Wc=
 github.com/libp2p/go-yamux/v4 v4.0.1-0.20220919134236-1c09f2ab3ec1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4=

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "starpc",
-  "version": "0.18.3",
+  "version": "0.19.1",
   "description": "Streaming protobuf RPC service protocol over any two-way channel.",
   "license": "MIT",
   "author": {
@@ -60,37 +60,37 @@
     "singleQuote": true
   },
   "devDependencies": {
-    "@aperturerobotics/ts-common": "^0.3.4",
-    "@typescript-eslint/eslint-plugin": "^5.56.0",
-    "@typescript-eslint/parser": "^5.56.0",
+    "@aperturerobotics/ts-common": "^0.5.0",
+    "@typescript-eslint/eslint-plugin": "^5.59.0",
+    "@typescript-eslint/parser": "^5.59.0",
     "bufferutil": "^4.0.7",
     "depcheck": "^1.4.3",
-    "esbuild": "^0.17.12",
-    "eslint": "^8.36.0",
+    "esbuild": "^0.17.17",
+    "eslint": "^8.38.0",
     "eslint-config-prettier": "^8.8.0",
-    "prettier": "^2.8.5",
-    "rimraf": "^4.4.0",
-    "ts-proto": "^1.143.0",
-    "typescript": "^5.0.2",
+    "prettier": "^2.8.7",
+    "rimraf": "^5.0.0",
+    "ts-proto": "^1.146.0",
+    "typescript": "^5.0.4",
     "utf-8-validate": "^6.0.3"
   },
   "dependencies": {
-    "@chainsafe/libp2p-yamux": "^3.0.7",
+    "@chainsafe/libp2p-yamux": "^3.0.10",
     "@libp2p/interface-connection": "^3.1.1",
     "@libp2p/interface-stream-muxer": "^3.0.4",
     "event-iterator": "^2.0.0",
     "is-promise": "^4.0.0",
     "isomorphic-ws": "^5.0.0",
     "it-first": "^2.0.1",
-    "it-length-prefixed": "^8.0.4",
-    "it-pipe": "^2.0.5",
+    "it-length-prefixed": "^8.0.0",
+    "it-pipe": "^2.0.0",
     "it-pushable": "^3.1.2",
     "it-stream-types": "^1.0.5",
-    "it-ws": "^5.0.6",
-    "long": "^5.2.1",
+    "it-ws": "^6.0.1",
+    "long": "^5.2.3",
     "memoize-one": "^6.0.0",
     "patch-package": "^6.5.1",
-    "protobufjs": "^7.1.2",
+    "protobufjs": "^7.2.0",
     "uint8arraylist": "^2.4.3",
     "ws": "^8.13.0"
   }

package/srpc/client-prefix.go

@@ -54,5 +54,12 @@ func (i *PrefixClient) stripCheckServiceIDPrefix(service string) (string, error)
 	return service, nil
 }
 
+// NewRawStream opens a new raw stream with the remote.
+// Implements OpenStreamFunc.
+// msgHandler must not be called concurrently.
+func (i *PrefixClient) NewRawStream(ctx context.Context, msgHandler PacketDataHandler, closeHandler CloseHandler) (Writer, error) {
+	return i.client.NewRawStream(ctx, msgHandler, closeHandler)
+}
+
 // _ is a type assertion
 var _ Client = ((*PrefixClient)(nil))

package/srpc/client-set.go

@@ -2,8 +2,6 @@ package srpc
 
 import (
 	"context"
-
-	"github.com/pkg/errors"
 )
 
 // ClientSet wraps a list of clients into one Client.
@@ -43,6 +41,23 @@ func (c *ClientSet) NewStream(
 	return strm, err
 }
 
+// NewRawStream opens a new raw stream with the remote.
+// Implements OpenStreamFunc.
+// msgHandler must not be called concurrently.
+func (c *ClientSet) NewRawStream(
+	ctx context.Context,
+	msgHandler PacketDataHandler,
+	closeHandler CloseHandler,
+) (Writer, error) {
+	for _, client := range c.clients {
+		if client == nil {
+			continue
+		}
+		return client.NewRawStream(ctx, msgHandler, closeHandler)
+	}
+	return nil, ErrNoAvailableClients
+}
+
 // execCall executes the call conditionally retrying against subsequent client handles.
 func (c *ClientSet) execCall(ctx context.Context, doCall func(client Client) error) error {
 	var any bool
@@ -70,7 +85,7 @@ func (c *ClientSet) execCall(ctx context.Context, doCall func(client Client) err
 	}
 
 	if !any {
-		return errors.New("no available rpc clients")
+		return ErrNoAvailableClients
 	}
 
 	return ErrUnimplemented

package/srpc/client.go

@@ -14,13 +14,18 @@ type Client interface {
 	// NewStream starts a streaming RPC with the remote & returns the stream.
 	// firstMsg is optional.
 	NewStream(ctx context.Context, service, method string, firstMsg Message) (Stream, error)
+
+	// NewRawStream opens a new raw stream with the remote.
+	// Implements OpenStreamFunc.
+	// msgHandler must not be called concurrently.
+	NewRawStream(ctx context.Context, msgHandler PacketDataHandler, closeHandler CloseHandler) (Writer, error)
 }
 
 // OpenStreamFunc opens a stream with a remote.
 // msgHandler must not be called concurrently.
 type OpenStreamFunc = func(
 	ctx context.Context,
-	msgHandler PacketHandler,
+	msgHandler PacketDataHandler,
 	closeHandler CloseHandler,
 ) (Writer, error)
 
@@ -47,7 +52,7 @@ func (c *client) ExecCall(ctx context.Context, service, method string, in, out M
 	clientRPC := NewClientRPC(ctx, service, method)
 	defer clientRPC.Close()
 
-	writer, err := c.openStream(ctx, clientRPC.HandlePacket, clientRPC.HandleStreamClose)
+	writer, err := c.openStream(ctx, clientRPC.HandlePacketData, clientRPC.HandleStreamClose)
 	if err != nil {
 		return err
 	}
@@ -79,7 +84,7 @@ func (c *client) NewStream(ctx context.Context, service, method string, firstMsg
 	}
 
 	clientRPC := NewClientRPC(ctx, service, method)
-	writer, err := c.openStream(ctx, clientRPC.HandlePacket, clientRPC.HandleStreamClose)
+	writer, err := c.openStream(ctx, clientRPC.HandlePacketData, clientRPC.HandleStreamClose)
 	if err != nil {
 		return nil, err
 	}
@@ -90,5 +95,16 @@ func (c *client) NewStream(ctx context.Context, service, method string, firstMsg
 	return NewMsgStream(ctx, clientRPC, clientRPC.ctxCancel), nil
 }
 
+// NewRawStream opens a new raw stream with the remote.
+// Implements OpenStreamFunc.
+// msgHandler must not be called concurrently.
+func (c *client) NewRawStream(
+	ctx context.Context,
+	msgHandler PacketDataHandler,
+	closeHandler CloseHandler,
+) (Writer, error) {
+	return c.openStream(ctx, msgHandler, closeHandler)
+}
+
 // _ is a type assertion
 var _ Client = ((*client)(nil))

package/srpc/client.ts

@@ -9,10 +9,7 @@ import { writeToPushable } from './pushable.js'
 import {
   decodePacketSource,
   encodePacketSource,
-  parseLengthPrefixTransform,
-  prependLengthPrefixTransform,
 } from './packet.js'
-import { combineUint8ArrayListTransform } from './array-list.js'
 import { OpenStreamCtr } from './open-stream-ctr.js'
 
 // Client implements the ts-proto Rpc interface with the drpcproto protocol.
@@ -117,20 +114,17 @@ export class Client implements TsProtoRpc {
       throw new Error(ERR_RPC_ABORT)
     }
     const openStreamFn = await this.openStreamCtr.wait()
-    const conn = await openStreamFn()
+    const stream = await openStreamFn()
     const call = new ClientRPC(rpcService, rpcMethod)
     abortSignal?.addEventListener('abort', () => {
       call.close(new Error(ERR_RPC_ABORT))
     })
     pipe(
-      conn,
-      parseLengthPrefixTransform(),
-      combineUint8ArrayListTransform(),
+      stream,
       decodePacketSource,
       call,
       encodePacketSource,
-      prependLengthPrefixTransform(),
-      conn
+      stream
     )
     await call.writeCallStart(data || undefined)
     return call

package/srpc/common-rpc.ts

@@ -113,7 +113,7 @@ export class CommonRPC {
         asError = new Error('error handling packet')
       }
       this.close(asError)
-      throw asError
+      // throw asError
     }
   }
 

package/srpc/conn.ts

@@ -8,10 +8,13 @@ import type { Duplex } from 'it-stream-types'
 import { yamux } from '@chainsafe/libp2p-yamux'
 import { Uint8ArrayList } from 'uint8arraylist'
 import isPromise from 'is-promise'
+import { pushable, Pushable } from 'it-pushable'
 
 import type { OpenStreamFunc, Stream as SRPCStream } from './stream.js'
 import { Client } from './client.js'
 import { combineUint8ArrayListTransform } from './array-list.js'
+import { parseLengthPrefixTransform, prependLengthPrefixTransform } from './packet.js'
+import { buildPushableSink } from './pushable.js'
 
 // ConnParams are parameters that can be passed to the Conn constructor.
 export interface ConnParams {
@@ -25,17 +28,25 @@ export interface ConnParams {
 // StreamHandler handles incoming streams.
 // Implemented by Server.
 export interface StreamHandler {
-  // handleStream handles an incoming stream.
-  handleStream(strm: SRPCStream): void
+  // handlePacketStream handles an incoming Uint8Array duplex.
+  // the stream has one Uint8Array per packet w/o length prefix.
+  handlePacketStream(strm: SRPCStream): void
 }
 
 // streamToSRPCStream converts a Stream to a SRPCStream.
+// uses length-prefix for packet framing
 export function streamToSRPCStream(
   stream: Duplex<Uint8ArrayList, Uint8ArrayList | Uint8Array>
 ): SRPCStream {
+  const pushSink: Pushable<Uint8Array> = pushable({ objectMode: true })
+  pipe(pushSink, prependLengthPrefixTransform(), stream.sink)
   return {
-    source: pipe(stream, combineUint8ArrayListTransform()),
-    sink: stream.sink,
+    source: pipe(
+      stream,
+      parseLengthPrefixTransform(),
+      combineUint8ArrayListTransform(),
+    ),
+    sink: buildPushableSink(pushSink),
   }
 }
 
@@ -85,9 +96,12 @@ export class Conn implements Duplex<Uint8Array> {
 
   // openStream implements the client open stream function.
   public async openStream(): Promise<SRPCStream> {
-    const stream = this.muxer.newStream()
-    if (isPromise(stream)) {
-      return streamToSRPCStream(await stream)
+    const streamPromise = this.muxer.newStream()
+    let stream: Stream
+    if (isPromise(streamPromise)) {
+      stream = await streamPromise
+    } else {
+      stream = streamPromise
     }
     return streamToSRPCStream(stream)
   }
@@ -103,6 +117,6 @@ export class Conn implements Duplex<Uint8Array> {
     if (!server) {
       return strm.abort(new Error('server not implemented'))
     }
-    server.handleStream(streamToSRPCStream(strm))
+    server.handlePacketStream(streamToSRPCStream(strm))
   }
 }

package/srpc/errors.go

@@ -17,4 +17,6 @@ var (
 	ErrEmptyMethodID = errors.New("method id empty")
 	// ErrEmptyServiceID is returned if the service id was empty.
 	ErrEmptyServiceID = errors.New("service id empty")
+	// ErrNoAvailableClients is returned if no clients were available.
+	ErrNoAvailableClients = errors.New("no available rpc clients")
 )

package/srpc/errors.ts

@@ -9,3 +9,27 @@ export function isAbortError(err: unknown): boolean {
   const message = (err as Error).message
   return message === ERR_RPC_ABORT
 }
+
+// castToError casts an object to an Error.
+// if err is a string, uses it as the message.
+// if err is undefined, returns new Error(defaultMsg)
+export function castToError(err: any, defaultMsg?: string): Error {
+  defaultMsg = defaultMsg || 'error'
+  if (!err) {
+    return new Error(defaultMsg)
+  }
+  if (typeof err === 'string') {
+    return new Error(err)
+  }
+  const asError = err as Error
+  if (asError.message) {
+    return asError
+  }
+  if (err.toString) {
+    const errString = err.toString()
+    if (errString) {
+      return new Error(errString)
+    }
+  }
+  return new Error(defaultMsg)
+}

package/srpc/index.ts

@@ -1,5 +1,5 @@
 export type { PacketHandler, Stream, OpenStreamFunc } from './stream.js'
-export { ERR_RPC_ABORT, isAbortError } from './errors.js'
+export { ERR_RPC_ABORT, isAbortError, castToError } from './errors.js'
 export { Client } from './client.js'
 export { Server } from './server.js'
 export { Conn, ConnParams } from './conn.js'
@@ -25,6 +25,13 @@ export {
   memoProto,
   memoProtoDecode,
 } from './message.js'
+export {
+  parseLengthPrefixTransform,
+  prependLengthPrefixTransform,
+  decodePacketSource,
+  encodePacketSource,
+} from './packet.js'
+export { combineUint8ArrayListTransform } from './array-list.js'
 export { ValueCtr } from './value-ctr.js'
 export { OpenStreamCtr } from './open-stream-ctr.js'
-export { writeToPushable } from './pushable'
+export { writeToPushable, buildPushableSink } from './pushable.js'

package/srpc/message.go

@@ -7,6 +7,8 @@ type Message interface {
 }
 
 // RawMessage is a raw protobuf message container.
+//
+// The empty value is valid with copy=false.
 type RawMessage struct {
 	data []byte
 	copy bool
@@ -41,6 +43,16 @@ func (m *RawMessage) SetData(data []byte) {
 	}
 }
 
+// Clear sets the length of the data buffer to 0 without releasing it.
+func (m *RawMessage) Clear() {
+	m.data = m.data[:0]
+}
+
+// Reset releases the data buffer.
+func (m *RawMessage) Reset() {
+	m.data = nil
+}
+
 func (m *RawMessage) MarshalVT() ([]byte, error) {
 	if !m.copy {
 		return m.data, nil

package/srpc/muxed-conn.go

@@ -72,7 +72,7 @@ func NewClientWithMuxedConn(conn network.MuxedConn) Client {
 
 // NewOpenStreamWithMuxedConn constructs a OpenStream func with a MuxedConn.
 func NewOpenStreamWithMuxedConn(conn network.MuxedConn) OpenStreamFunc {
-	return func(ctx context.Context, msgHandler PacketHandler, closeHandler CloseHandler) (Writer, error) {
+	return func(ctx context.Context, msgHandler PacketDataHandler, closeHandler CloseHandler) (Writer, error) {
 		mstrm, err := conn.OpenStream(ctx)
 		if err != nil {
 			return nil, err

package/srpc/packet-rw.go

@@ -13,9 +13,9 @@ import (
 // maxMessageSize is the max message size in bytes
 var maxMessageSize = 1e7
 
-// PacketReaderWriter reads and writes packets from a io.ReadWriter.
+// PacketReadWriter reads and writes packets from a io.ReadWriter.
 // Uses a LittleEndian uint32 length prefix.
-type PacketReaderWriter struct {
+type PacketReadWriter struct {
 	// rw is the io.ReadWriterCloser
 	rw io.ReadWriteCloser
 	// buf is the buffered data
@@ -25,12 +25,19 @@ type PacketReaderWriter struct {
 }
 
 // NewPacketReadWriter constructs a new read/writer.
-func NewPacketReadWriter(rw io.ReadWriteCloser) *PacketReaderWriter {
-	return &PacketReaderWriter{rw: rw}
+func NewPacketReadWriter(rw io.ReadWriteCloser) *PacketReadWriter {
+	return &PacketReadWriter{rw: rw}
+}
+
+// Write writes raw data to the remote.
+func (r *PacketReadWriter) Write(p []byte) (n int, err error) {
+	r.writeMtx.Lock()
+	defer r.writeMtx.Unlock()
+	return r.rw.Write(p)
 }
 
 // WritePacket writes a packet to the writer.
-func (r *PacketReaderWriter) WritePacket(p *Packet) error {
+func (r *PacketReadWriter) WritePacket(p *Packet) error {
 	r.writeMtx.Lock()
 	defer r.writeMtx.Unlock()
 
@@ -55,7 +62,7 @@ func (r *PacketReaderWriter) WritePacket(p *Packet) error {
 // ReadPump executes the read pump in a goroutine.
 //
 // calls the handler when closed or returning an error
-func (r *PacketReaderWriter) ReadPump(cb PacketHandler, closed CloseHandler) {
+func (r *PacketReadWriter) ReadPump(cb PacketDataHandler, closed CloseHandler) {
 	err := r.ReadToHandler(cb)
 	// signal that the stream is now closed.
 	if closed != nil {
@@ -65,7 +72,7 @@ func (r *PacketReaderWriter) ReadPump(cb PacketHandler, closed CloseHandler) {
 
 // ReadToHandler reads data to the given handler.
 // Does not handle closing the stream, use ReadPump instead.
-func (r *PacketReaderWriter) ReadToHandler(cb PacketHandler) error {
+func (r *PacketReadWriter) ReadToHandler(cb PacketDataHandler) error {
 	var currLen uint32
 	buf := make([]byte, 2048)
 	isOpen := true
@@ -107,11 +114,7 @@ func (r *PacketReaderWriter) ReadToHandler(cb PacketHandler) error {
 		if currLen != 0 && bufLen >= int(currLen)+4 {
 			pkt := r.buf.Next(int(currLen + 4))[4:]
 			currLen = 0
-			npkt := &Packet{}
-			if err := npkt.UnmarshalVT(pkt); err != nil {
-				return err
-			}
-			if err := cb(npkt); err != nil {
+			if err := cb(pkt); err != nil {
 				return err
 			}
 		}
@@ -122,12 +125,12 @@ func (r *PacketReaderWriter) ReadToHandler(cb PacketHandler) error {
 }
 
 // Close closes the packet rw.
-func (r *PacketReaderWriter) Close() error {
+func (r *PacketReadWriter) Close() error {
 	return r.rw.Close()
 }
 
 // readLengthPrefix reads the length prefix.
-func (r *PacketReaderWriter) readLengthPrefix(b []byte) uint32 {
+func (r *PacketReadWriter) readLengthPrefix(b []byte) uint32 {
 	if len(b) < 4 {
 		return 0
 	}
@@ -135,4 +138,4 @@ func (r *PacketReaderWriter) readLengthPrefix(b []byte) uint32 {
 }
 
 // _ is a type assertion
-var _ Writer = (*PacketReaderWriter)(nil)
+var _ Writer = (*PacketReadWriter)(nil)

package/srpc/packet.go

@@ -1,14 +1,33 @@
 package srpc
 
+import "context"
+
 // PacketHandler handles a packet.
 //
 // pkt is optional (can be nil)
 // if closeErr is set, the stream is closed after pkt.
 type PacketHandler = func(pkt *Packet) error
 
+// PacketDataHandler handles a packet before it is parsed.
+type PacketDataHandler = func(data []byte) error
+
 // CloseHandler handles the stream closing with an optional error.
 type CloseHandler = func(closeErr error)
 
+// RawStreamCtor is a function that builds a raw stream.
+type RawStreamCtor func(ctx context.Context, msgHandler PacketDataHandler, closeHandler CloseHandler) (Writer, error)
+
+// NewPacketDataHandler wraps a PacketHandler with a decoding step.
+func NewPacketDataHandler(handler PacketHandler) PacketDataHandler {
+	return func(data []byte) error {
+		pkt := &Packet{}
+		if err := pkt.UnmarshalVT(data); err != nil {
+			return err
+		}
+		return handler(pkt)
+	}
+}
+
 // Validate performs cursory validation of the packet.
 func (p *Packet) Validate() error {
 	switch b := p.GetBody().(type) {

package/srpc/pushable.ts

@@ -1,4 +1,6 @@
 import { Pushable } from 'it-pushable'
+import { Source, Sink } from 'it-stream-types'
+import { castToError } from './errors.js'
 
 // writeToPushable writes the incoming server data to the pushable.
 export async function writeToPushable<T>(
@@ -15,3 +17,24 @@ export async function writeToPushable<T>(
     throw err
   }
 }
+
+// buildPushableSink builds a Sink from a Pushable.
+export function buildPushableSink<T>(target: Pushable<T>): Sink<T> {
+  return async function pushableSink(source: Source<T>): Promise<void> {
+    try {
+      for await (const pkt of source) {
+        if (Array.isArray(pkt)) {
+          for (const p of pkt) {
+            target.push(p)
+          }
+        } else {
+          target.push(pkt)
+        }
+      }
+      target.end()
+    } catch (err) {
+      const error = castToError(err)
+      target.end(error)
+    }
+  }
+}

package/srpc/raw-stream-rwc.go

@@ -0,0 +1,153 @@
+package srpc
+
+import (
+	"context"
+	"io"
+	"sync"
+
+	"github.com/aperturerobotics/util/broadcast"
+)
+
+// RawStreamRwc implements io.ReadWriteCloser with a raw stream.
+type RawStreamRwc struct {
+	// writer is used to write messages to the remote.
+	writer Writer
+	// mtx guards below fields
+	mtx sync.Mutex
+	// bcast is broadcasted when data is added to readQueue
+	bcast broadcast.Broadcast
+	// readQueue holds incoming data to read
+	readQueue [][]byte
+	// closed indicates whether the stream is closed
+	closed bool
+	// closeErr stores the error, if any, when closing the stream
+	closeErr error
+}
+
+func NewRawStreamRwc(ctx context.Context, ctorFn RawStreamCtor) (*RawStreamRwc, error) {
+	rwc := &RawStreamRwc{}
+	var err error
+	rwc.writer, err = ctorFn(ctx, rwc.handlePacketData, rwc.handleClose)
+	if err != nil {
+		return nil, err
+	}
+	return rwc, nil
+}
+
+// handlePacketData implements PacketDataHandler.
+func (r *RawStreamRwc) handlePacketData(pkt []byte) error {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	if r.closed {
+		return io.ErrClosedPipe
+	}
+
+	r.readQueue = append(r.readQueue, pkt)
+	r.bcast.Broadcast()
+	return nil
+}
+
+// handleClose handles the stream closing with an optional error.
+func (r *RawStreamRwc) handleClose(closeErr error) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	if r.closed {
+		return
+	}
+	r.closed = true
+	r.closeErr = closeErr
+	r.bcast.Broadcast()
+}
+
+// Read reads data from the stream to p.
+// Implements io.Reader.
+func (r *RawStreamRwc) Read(p []byte) (n int, err error) {
+	readBuf := p
+	for len(readBuf) != 0 && err == nil {
+		// if the buffer has data, read from it.
+		var rn int
+		var read []byte
+
+		r.mtx.Lock()
+		if len(r.readQueue) != 0 {
+			nrq := r.readQueue[0]
+			rn = len(readBuf)
+			if nrLen := len(nrq); nrLen < rn {
+				rn = nrLen
+			}
+			read = nrq[:rn]
+			nrq = nrq[rn:]
+			if len(nrq) == 0 {
+				r.readQueue[0] = nil
+				r.readQueue = r.readQueue[1:]
+			} else {
+				r.readQueue[0] = nrq
+			}
+		}
+		closed, closedErr := r.closed, r.closeErr
+		var wait <-chan struct{}
+		if rn == 0 && !closed {
+			wait = r.bcast.GetWaitCh()
+		}
+		r.mtx.Unlock()
+
+		// if we read data, copy it to the output buf
+		if rn != 0 {
+			// advance readBuf by rn
+			copy(readBuf, read)
+			n += rn
+			readBuf = readBuf[rn:]
+			continue
+		}
+
+		// if we read data to p already, return now.
+		if n != 0 {
+			break
+		}
+
+		// if closed or error, return.
+		if closed {
+			if closedErr != nil {
+				return n, closedErr
+			}
+			return n, io.EOF
+		}
+
+		// wait for data or closed
+		<-wait
+	}
+
+	return n, err
+}
+
+// Write writes data to the stream.
+func (r *RawStreamRwc) Write(p []byte) (int, error) {
+	return r.writer.Write(p)
+}
+
+// WritePacket writes a packet to the remote.
+func (r *RawStreamRwc) WritePacket(p *Packet) error {
+	return r.writer.WritePacket(p)
+}
+
+// Close closes the stream.
+func (r *RawStreamRwc) Close() error {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	if r.closed {
+		return r.closeErr
+	}
+
+	r.closed = true
+	r.closeErr = r.writer.Close()
+	r.bcast.Broadcast()
+	return r.closeErr
+}
+
+// _ is a type assertion
+var (
+	_ io.ReadWriteCloser = ((*RawStreamRwc)(nil))
+	_ Writer             = ((*RawStreamRwc)(nil))
+)

package/srpc/server-pipe.go

@@ -9,7 +9,7 @@ import (
 // Stream with the given Server. Starts read pumps for both. Starts the
 // HandleStream function on the server in a separate goroutine.
 func NewServerPipe(server *Server) OpenStreamFunc {
-	return func(ctx context.Context, msgHandler PacketHandler, closeHandler CloseHandler) (Writer, error) {
+	return func(ctx context.Context, msgHandler PacketDataHandler, closeHandler CloseHandler) (Writer, error) {
 		srvPipe, clientPipe := net.Pipe()
 		go server.HandleStream(ctx, srvPipe)
 		clientPrw := NewPacketReadWriter(clientPipe)

package/srpc/server-rpc.go

@@ -22,6 +22,15 @@ func NewServerRPC(ctx context.Context, invoker Invoker, writer Writer) *ServerRP
 	return rpc
 }
 
+// HandlePacketData handles an incoming unparsed message packet.
+func (r *ServerRPC) HandlePacketData(data []byte) error {
+	msg := &Packet{}
+	if err := msg.UnmarshalVT(data); err != nil {
+		return err
+	}
+	return r.HandlePacket(msg)
+}
+
 // HandlePacket handles an incoming parsed message packet.
 func (r *ServerRPC) HandlePacket(msg *Packet) error {
 	if msg == nil {

package/srpc/server.go

@@ -26,12 +26,13 @@ func (s *Server) GetInvoker() Invoker {
 }
 
 // HandleStream handles an incoming stream and runs the read loop.
+// Uses length-prefixed packets.
 func (s *Server) HandleStream(ctx context.Context, rwc io.ReadWriteCloser) {
 	subCtx, subCtxCancel := context.WithCancel(ctx)
 	defer subCtxCancel()
 	prw := NewPacketReadWriter(rwc)
 	serverRPC := NewServerRPC(subCtx, s.invoker, prw)
-	prw.ReadPump(serverRPC.HandlePacket, serverRPC.HandleStreamClose)
+	prw.ReadPump(serverRPC.HandlePacketData, serverRPC.HandleStreamClose)
 }
 
 // AcceptMuxedConn runs a loop which calls Accept on a muxer to handle streams.

package/srpc/server.ts

@@ -25,8 +25,9 @@ export class Server implements StreamHandler {
   }
 
   // rpcStreamHandler implements the RpcStreamHandler interface.
+  // uses handlePacketDuplex (expects 1 buf = 1 Packet)
   public get rpcStreamHandler(): RpcStreamHandler {
-    return this.handleStream.bind(this)
+    return this.handlePacketStream.bind(this)
   }
 
   // startRpc starts a new server-side RPC.
@@ -35,13 +36,10 @@ export class Server implements StreamHandler {
     return new ServerRPC(this.lookupMethod)
   }
 
-  // handleStream handles an incoming Uint8Array message duplex.
-  public handleStream(stream: Stream): ServerRPC {
-    return this.handleDuplex(stream)
-  }
-
-  // handleDuplex handles an incoming message duplex.
-  public handleDuplex(stream: Duplex<Uint8Array, Uint8Array>): ServerRPC {
+  // handleFragmentStream handles an incoming stream.
+  // assumes that stream does not maintain packet framing.
+  // uses length-prefixed packets for packet framing.
+  public handleFragmentStream(stream: Stream): ServerRPC {
     const rpc = this.startRpc()
     pipe(
       stream,
@@ -57,16 +55,16 @@ export class Server implements StreamHandler {
     return rpc
   }
 
-  // handlePacketDuplex handles an incoming Uint8Array duplex.
-  // skips the packet length prefix transform.
-  public handlePacketDuplex(stream: Duplex<Uint8Array>): ServerRPC {
+  // handlePacketStream handles an incoming Uint8Array duplex.
+  // the stream has one Uint8Array per packet w/o length prefix.
+  public handlePacketStream(stream: Stream): ServerRPC {
     const rpc = this.startRpc()
     pipe(stream, decodePacketSource, rpc, encodePacketSource, stream)
     return rpc
   }
 
-  // handlePacketStream handles an incoming Packet duplex.
-  public handlePacketStream(stream: Duplex<Packet>): ServerRPC {
+  // handlePacketDuplex handles an incoming Packet duplex.
+  public handlePacketDuplex(stream: Duplex<Packet>): ServerRPC {
     const rpc = this.startRpc()
     pipe(stream, rpc, stream)
     return rpc

package/srpc/stream-rwc.go

@@ -0,0 +1,85 @@
+package srpc
+
+import (
+	"bytes"
+	"io"
+)
+
+// StreamRwc implements an io.ReadWriteCloser with a srpc.Stream.
+type StreamRwc struct {
+	// Stream is the base stream interface.
+	Stream
+
+	// buf is the incoming data buffer
+	buf bytes.Buffer
+	// readMsg is the raw read message
+	readMsg RawMessage
+	// writeMsg is the raw write message
+	writeMsg RawMessage
+}
+
+// NewStreamRwc constructs a new stream read write closer.
+func NewStreamRwc(strm Stream) *StreamRwc {
+	rwc := &StreamRwc{Stream: strm}
+	rwc.readMsg.copy = true
+	rwc.writeMsg.copy = true
+	return rwc
+}
+
+// Read reads data from the stream to p.
+// Implements io.Reader.
+func (s *StreamRwc) Read(p []byte) (n int, err error) {
+	readBuf := p
+	for len(readBuf) != 0 && err == nil {
+		var rn int
+
+		// if the buffer has data, read from it.
+		if s.buf.Len() != 0 {
+			rn, err = s.buf.Read(readBuf)
+		} else {
+			if n != 0 {
+				// if we read data to p already, return now.
+				break
+			}
+
+			s.readMsg.Clear()
+			if err := s.Stream.MsgRecv(&s.readMsg); err != nil {
+				return n, err
+			}
+			data := s.readMsg.GetData()
+			if len(data) == 0 {
+				continue
+			}
+
+			// read as much as possible directly to the output
+			copy(readBuf, data)
+			if len(data) > len(readBuf) {
+				// we read some of the data, buffer the rest.
+				rn = len(readBuf)
+				_, _ = s.buf.Write(data[rn:]) // never returns an error
+			} else {
+				// we read all of data
+				rn = len(data)
+			}
+		}
+
+		// advance readBuf by rn
+		n += rn
+		readBuf = readBuf[rn:]
+	}
+	return n, err
+}
+
+// Write writes data to the stream.
+func (s *StreamRwc) Write(p []byte) (n int, err error) {
+	s.writeMsg.SetData(p)
+	err = s.Stream.MsgSend(&s.writeMsg)
+	s.writeMsg.Clear()
+	if err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
+
+// _ is a type assertion
+var _ io.ReadWriteCloser = ((*StreamRwc)(nil))

package/srpc/stream.ts

@@ -5,6 +5,7 @@ import type { Duplex } from 'it-stream-types'
 export type PacketHandler = (packet: Packet) => Promise<void>
 
 // Stream is an open connection.
+// This stream type generally assumes that each Uint8Array corresponds to a Packet.
 export type Stream = Duplex<Uint8Array>
 
 // OpenStreamFunc is a function to start a new RPC by opening a Stream.

package/srpc/writer.go

@@ -2,6 +2,8 @@ package srpc
 
 // Writer is the interface used to write messages to the remote.
 type Writer interface {
+	// Write writes raw data to the remote.
+	Write(p []byte) (n int, err error)
 	// WritePacket writes a packet to the remote.
 	WritePacket(p *Packet) error
 	// Close closes the writer.