starpc 0.0.1 → 0.1.0

package/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2022 Aperture Robotics, LLC.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,56 +1,151 @@
-# Stream RPC (starpc)
+# Stream RPC
 
-**starpc** is a Protobuf RPC implementation leveraging an external Stream
-multiplexer and Message framing system like HTTP/2 or Yamux.
+**starpc** is a fully-featured client and server for [Proto3 services] in both
+TypeScript and Go.
 
-Uses any two-way ordered message channel like WebSocket or BroadcastChannel.
+[Proto3 services]: https://developers.google.com/protocol-buffers/docs/proto3#services
 
-Implementation features:
+One of the first libraries to support **client-to-server streaming** RPCs.
 
- - Generates a Go server for proto3 services that does not use reflection.
- - Uses the **ts-proto** RPC interface to implement TypeScript clients.
- - Each RPC call is mapped to a Stream preventing head-of-line blocking.
+The [rpcproto](./srpc/rpcproto.proto) file contains the entire protocol.
 
-The Go backend provides an interface for forwarding the incoming RPC calls to
-the appropriate backend service.
+Leverages the Stream multiplexing of the underlying transport; for example:
+HTTP/2 or [libp2p-mplex] over a WebSocket.
 
-# Usage
+[libp2p-mplex]: https://github.com/libp2p/js-libp2p-mplex
 
-## Go
+The server side has not yet been implemented in TypeScript.
+
+# Examples
+
+See the [protobuf-project] template on the "starpc" branch.
+
+[protobuf-project]: https://github.com/aperturerobotics/protobuf-project
 
-TODO
+## Protobuf
 
-Copy the Go project boilerplate files:
+The following examples use the [echo](./echo/echo.proto) protobuf sample.
 
- - Makefile: implements "make gengo" to generate Go code with protoc.
- - hack/: downloads & builds the necessary protoc-gen-go tools.
+```protobuf
+syntax = "proto3";
+package echo;
 
-To generate code for .proto files in your repository:
+// Echoer service returns the given message.
+service Echoer {
+  // Echo returns the given message.
+  rpc Echo(EchoMsg) returns (EchoMsg);
+  // EchoServerStream is an example of a server -> client one-way stream.
+  rpc EchoServerStream(EchoMsg) returns (stream EchoMsg);
+  // EchoClientStream is an example of client->server one-way stream.
+  rpc EchoClientStream(stream EchoMsg) returns (EchoMsg);
+  // EchoBidiStream is an example of a two-way stream.
+  rpc EchoBidiStream(stream EchoMsg) returns (stream EchoMsg);
+}
 
- - `git add .` - stage all changes first.
- - `make gengo` - generate the protobuf files.
+// EchoMsg is the message body for Echo.
+message EchoMsg {
+  string body = 1;
+}
+```
 
-Example of creating & serving a service in Go: (TODO)
+## Go
+
+A basic example can be found in the [e2e test]:
+
+```go
+// construct the server
+echoServer := &echo.EchoServer{}
+mux := srpc.NewMux()
+if err := echo.SRPCRegisterEchoer(mux, echoServer); err != nil {
+	t.Fatal(err.Error())
+}
+server := srpc.NewServer(mux)
+
+// create an in-memory connection to the server
+openStream := srpc.NewServerPipe(server)
+
+// construct the client
+client := srpc.NewClient(openStream)
+
+// construct the client rpc interface
+clientEcho := echo.NewSRPCEchoerClient(client)
+ctx := context.Background()
+bodyTxt := "hello world"
+out, err := clientEcho.Echo(ctx, &echo.EchoMsg{
+	Body: bodyTxt,
+})
+if err != nil {
+	t.Fatal(err.Error())
+}
+if out.GetBody() != bodyTxt {
+	t.Fatalf("expected %q got %q", bodyTxt, out.GetBody())
+}
+```
+
+[e2e test]: ./e2e/e2e_test.go
 
 ## TypeScript
 
 See the ts-proto README to generate the TypeScript for your protobufs.
 
+Also check out the [integration](./integration/integration.ts) test.
+
+Supports any AsyncIterable communication channel with an included implementation
+for WebSockets.
+
+This repository uses protowrap, see the [Makefile](./Makefile).
+
 ```typescript
-import { RpcClient, WebSocketConn } from 'starpc'
-import { DemoServiceClientImpl } from './demo'
+import { WebSocketConn } from '../srpc'
+import { EchoerClientImpl } from '../echo/echo'
 
 const ws = new WebSocket('ws://localhost:5000/demo')
 const channel = new WebSocketConn(ws)
-const rpc = new RpcClient(channel)
-const demoServiceClient = new DemoServiceClientImpl(rpc)
+const client = channel.buildClient()
+const demoServiceClient = new EchoerClientImpl(client)
+
+const result = await demoServiceClient.Echo({
+  body: "Hello world!"
+})
+console.log('output', result.body)
 
-const result = await demoServiceClient.DemoEcho({
-  msg: "Hello world!"
+const clientRequestStream = new Observable<EchoMsg>(subscriber => {
+  subscriber.next({body: 'Hello world from streaming request.'})
+  subscriber.complete()
 })
-console.log('output', result.msg)
+
+console.log('Calling EchoClientStream: client -> server...')
+result = await demoServiceClient.EchoClientStream(clientRequestStream)
+console.log('success: output', result.body)
 ```
 
 `WebSocketConn` uses [js-libp2p-mplex] to multiplex streams over the WebSocket.
 
 [js-libp2p-mplex]: https://github.com/libp2p/js-libp2p-mplex
+
+# Attribution
+
+`protoc-gen-go-starpc` is a heavily modified version of `protoc-gen-go-drpc`.
+
+Be sure to check out [drpc] as well: it's compatible with grpc, twirp, and more.
+
+[drpc]: https://github.com/storj/drpc
+
+Uses [vtprotobuf] to generate Protobuf marshal / unmarshal code.
+
+[vtprotobuf]: https://github.com/planetscale/vtprotobuf
+
+# Support
+
+Starpc is built & supported by Aperture Robotics, LLC.
+
+Community contributions and discussion are welcomed!
+
+Please open a [GitHub issue] with any questions / issues.
+
+[GitHub issue]: https://github.com/aperturerobotics/bifrost/issues/new
+
+... or feel free to reach out on [Matrix Chat] or [Discord].
+
+[Discord]: https://discord.gg/KJutMESRsT
+[Matrix Chat]: https://matrix.to/#/#aperturerobotics:matrix.org

package/dist/echo/echo.d.ts

@@ -0,0 +1,59 @@
+import Long from 'long';
+import * as _m0 from 'protobufjs/minimal';
+import { Observable } from 'rxjs';
+export declare const protobufPackage = "echo";
+/** EchoMsg is the message body for Echo. */
+export interface EchoMsg {
+    body: string;
+}
+export declare const EchoMsg: {
+    encode(message: EchoMsg, writer?: _m0.Writer): _m0.Writer;
+    decode(input: _m0.Reader | Uint8Array, length?: number): EchoMsg;
+    fromJSON(object: any): EchoMsg;
+    toJSON(message: EchoMsg): unknown;
+    fromPartial<I extends {
+        body?: string | undefined;
+    } & {
+        body?: string | undefined;
+    } & Record<Exclude<keyof I, "body">, never>>(object: I): EchoMsg;
+};
+/** Echoer service returns the given message. */
+export interface Echoer {
+    /** Echo returns the given message. */
+    Echo(request: EchoMsg): Promise<EchoMsg>;
+    /** EchoServerStream is an example of a server -> client one-way stream. */
+    EchoServerStream(request: EchoMsg): Observable<EchoMsg>;
+    /** EchoClientStream is an example of client->server one-way stream. */
+    EchoClientStream(request: Observable<EchoMsg>): Promise<EchoMsg>;
+    /** EchoBidiStream is an example of a two-way stream. */
+    EchoBidiStream(request: Observable<EchoMsg>): Observable<EchoMsg>;
+}
+export declare class EchoerClientImpl implements Echoer {
+    private readonly rpc;
+    constructor(rpc: Rpc);
+    Echo(request: EchoMsg): Promise<EchoMsg>;
+    EchoServerStream(request: EchoMsg): Observable<EchoMsg>;
+    EchoClientStream(request: Observable<EchoMsg>): Promise<EchoMsg>;
+    EchoBidiStream(request: Observable<EchoMsg>): Observable<EchoMsg>;
+}
+interface Rpc {
+    request(service: string, method: string, data: Uint8Array): Promise<Uint8Array>;
+    clientStreamingRequest(service: string, method: string, data: Observable<Uint8Array>): Promise<Uint8Array>;
+    serverStreamingRequest(service: string, method: string, data: Uint8Array): Observable<Uint8Array>;
+    bidirectionalStreamingRequest(service: string, method: string, data: Observable<Uint8Array>): Observable<Uint8Array>;
+}
+declare type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
+export declare type DeepPartial<T> = T extends Builtin ? T : T extends Long ? string | number | Long : T extends Array<infer U> ? Array<DeepPartial<U>> : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>> : T extends {
+    $case: string;
+} ? {
+    [K in keyof Omit<T, '$case'>]?: DeepPartial<T[K]>;
+} & {
+    $case: T['$case'];
+} : T extends {} ? {
+    [K in keyof T]?: DeepPartial<T[K]>;
+} : Partial<T>;
+declare type KeysOfUnion<T> = T extends T ? keyof T : never;
+export declare type Exact<P, I extends P> = P extends Builtin ? P : P & {
+    [K in keyof P]: Exact<P[K], I[K]>;
+} & Record<Exclude<keyof I, KeysOfUnion<P>>, never>;
+export {};

package/dist/echo/echo.js

@@ -0,0 +1,85 @@
+/* eslint-disable */
+import Long from 'long';
+import * as _m0 from 'protobufjs/minimal';
+import { map } from 'rxjs/operators';
+export const protobufPackage = 'echo';
+function createBaseEchoMsg() {
+    return { body: '' };
+}
+export const EchoMsg = {
+    encode(message, writer = _m0.Writer.create()) {
+        if (message.body !== '') {
+            writer.uint32(10).string(message.body);
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseEchoMsg();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1:
+                    message.body = reader.string();
+                    break;
+                default:
+                    reader.skipType(tag & 7);
+                    break;
+            }
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            body: isSet(object.body) ? String(object.body) : '',
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        message.body !== undefined && (obj.body = message.body);
+        return obj;
+    },
+    fromPartial(object) {
+        const message = createBaseEchoMsg();
+        message.body = object.body ?? '';
+        return message;
+    },
+};
+export class EchoerClientImpl {
+    rpc;
+    constructor(rpc) {
+        this.rpc = rpc;
+        this.Echo = this.Echo.bind(this);
+        this.EchoServerStream = this.EchoServerStream.bind(this);
+        this.EchoClientStream = this.EchoClientStream.bind(this);
+        this.EchoBidiStream = this.EchoBidiStream.bind(this);
+    }
+    Echo(request) {
+        const data = EchoMsg.encode(request).finish();
+        const promise = this.rpc.request('echo.Echoer', 'Echo', data);
+        return promise.then((data) => EchoMsg.decode(new _m0.Reader(data)));
+    }
+    EchoServerStream(request) {
+        const data = EchoMsg.encode(request).finish();
+        const result = this.rpc.serverStreamingRequest('echo.Echoer', 'EchoServerStream', data);
+        return result.pipe(map((data) => EchoMsg.decode(new _m0.Reader(data))));
+    }
+    EchoClientStream(request) {
+        const data = request.pipe(map((request) => EchoMsg.encode(request).finish()));
+        const promise = this.rpc.clientStreamingRequest('echo.Echoer', 'EchoClientStream', data);
+        return promise.then((data) => EchoMsg.decode(new _m0.Reader(data)));
+    }
+    EchoBidiStream(request) {
+        const data = request.pipe(map((request) => EchoMsg.encode(request).finish()));
+        const result = this.rpc.bidirectionalStreamingRequest('echo.Echoer', 'EchoBidiStream', data);
+        return result.pipe(map((data) => EchoMsg.decode(new _m0.Reader(data))));
+    }
+}
+if (_m0.util.Long !== Long) {
+    _m0.util.Long = Long;
+    _m0.configure();
+}
+function isSet(value) {
+    return value !== null && value !== undefined;
+}

package/dist/srpc/broadcast-channel.d.ts

@@ -0,0 +1,16 @@
+import type { Duplex, Sink } from 'it-stream-types';
+import { Conn } from './conn';
+export declare class BroadcastChannelIterable<T> implements Duplex<T> {
+    readonly channel: BroadcastChannel;
+    sink: Sink<T>;
+    source: AsyncIterable<T>;
+    constructor(channel: BroadcastChannel);
+    private _createSink;
+    private _createSource;
+}
+export declare function newBroadcastChannelIterable<T>(name: string): BroadcastChannelIterable<T>;
+export declare class BroadcastChannelConn extends Conn {
+    private channel;
+    constructor(channel: BroadcastChannel);
+    getBroadcastChannel(): BroadcastChannel;
+}

package/dist/srpc/broadcast-channel.js

@@ -0,0 +1,60 @@
+import { Conn } from './conn';
+import { EventIterator } from 'event-iterator';
+import { pipe } from 'it-pipe';
+// BroadcastChannelIterable is a AsyncIterable wrapper for BroadcastChannel.
+export class BroadcastChannelIterable {
+    // channel is the broadcast channel
+    channel;
+    // sink is the sink for incoming messages.
+    sink;
+    // source is the source for outgoing messages.
+    source;
+    constructor(channel) {
+        this.channel = channel;
+        this.sink = this._createSink();
+        this.source = this._createSource();
+    }
+    // _createSink initializes the sink field.
+    _createSink() {
+        return async (source) => {
+            for await (const msg of source) {
+                this.channel.postMessage(msg);
+            }
+        };
+    }
+    // _createSource initializes the source field.
+    _createSource() {
+        return new EventIterator((queue) => {
+            const messageListener = (ev) => {
+                if (ev.data) {
+                    queue.push(ev.data);
+                }
+            };
+            this.channel.addEventListener('message', messageListener);
+            return () => {
+                this.channel.removeEventListener('message', messageListener);
+            };
+        });
+    }
+}
+// newBroadcastChannelIterable constructs a BroadcastChannelIterable with a channel name.
+export function newBroadcastChannelIterable(name) {
+    const channel = new BroadcastChannel(name);
+    return new BroadcastChannelIterable(channel);
+}
+// BroadcastChannelConn implements a connection with a BroadcastChannel.
+//
+// expects Uint8Array objects over the BroadcastChannel.
+export class BroadcastChannelConn extends Conn {
+    // channel is the broadcast channel iterable
+    channel;
+    constructor(channel) {
+        super();
+        this.channel = new BroadcastChannelIterable(channel);
+        pipe(this, this.channel, this);
+    }
+    // getBroadcastChannel returns the BroadcastChannel.
+    getBroadcastChannel() {
+        return this.channel.channel;
+    }
+}

package/dist/srpc/client-rpc.d.ts

@@ -0,0 +1,31 @@
+import type { CallData, CallStart, CallStartResp } from './rpcproto';
+import { Packet } from './rpcproto';
+import type { Sink } from 'it-stream-types';
+export declare type DataCb = (data: Uint8Array) => Promise<boolean | void>;
+export declare class ClientRPC {
+    sink: Sink<Packet>;
+    source: AsyncIterable<Packet>;
+    private readonly _source;
+    private service;
+    private method;
+    private dataCb?;
+    private started;
+    private onStarted?;
+    private complete;
+    private onComplete?;
+    private closed;
+    constructor(service: string, method: string, dataCb: DataCb | null);
+    waitStarted(): Promise<void>;
+    waitComplete(): Promise<void>;
+    writeCallStart(data?: Uint8Array): Promise<void>;
+    writeCallData(data: Uint8Array, complete?: boolean, error?: string): Promise<void>;
+    private writePacket;
+    handleMessage(message: Uint8Array): Promise<void>;
+    handlePacket(packet: Partial<Packet>): Promise<void>;
+    handleCallStart(packet: Partial<CallStart>): Promise<void>;
+    handleCallStartResp(packet: Partial<CallStartResp>): Promise<void>;
+    handleCallData(packet: Partial<CallData>): Promise<void>;
+    close(err?: Error): Promise<void>;
+    private _createSink;
+    private _createSource;
+}

package/dist/srpc/client-rpc.js

@@ -0,0 +1,176 @@
+import { Packet } from './rpcproto';
+import { pushable } from 'it-pushable';
+// ClientRPC is an ongoing RPC from the client side.
+export class ClientRPC {
+    // sink is the data sink for incoming messages.
+    sink;
+    // source is the packet source for outgoing Packets.
+    source;
+    // _source is used to write to the source.
+    _source;
+    // service is the rpc service
+    service;
+    // method is the rpc method
+    method;
+    // dataCb is called with any incoming data.
+    dataCb;
+    // started is resolved when the request starts.
+    started;
+    // onStarted is called by the message handler when the request starts.
+    onStarted;
+    // complete is resolved when the request completes.
+    // rejected with an error if the call encountered any error.
+    complete;
+    // onComplete is called by the message handler when the call completes.
+    onComplete;
+    // closed indicates close has been called
+    closed;
+    constructor(service, method, dataCb) {
+        this.closed = false;
+        this.sink = this._createSink();
+        const sourcev = this._createSource();
+        this.source = sourcev;
+        this._source = sourcev;
+        this.service = service;
+        this.method = method;
+        if (dataCb) {
+            this.dataCb = dataCb;
+        }
+        this.started = new Promise((resolveStarted, rejectStarted) => {
+            this.onStarted = (err) => {
+                if (err) {
+                    rejectStarted(err);
+                }
+                else {
+                    resolveStarted();
+                }
+            };
+        });
+        this.complete = new Promise((resolveComplete, rejectComplete) => {
+            this.onComplete = (err) => {
+                this.closed = true;
+                if (err) {
+                    rejectComplete(err);
+                }
+                else {
+                    resolveComplete();
+                }
+            };
+        });
+    }
+    // waitStarted returns the started promise.
+    waitStarted() {
+        return this.started;
+    }
+    // waitComplete returns the complete promise.
+    waitComplete() {
+        return this.complete;
+    }
+    // writeCallStart writes the call start packet.
+    async writeCallStart(data) {
+        const callStart = {
+            rpcService: this.service,
+            rpcMethod: this.method,
+            data: data || new Uint8Array(0),
+        };
+        await this.writePacket({
+            body: {
+                $case: 'callStart',
+                callStart,
+            },
+        });
+    }
+    // writeCallData writes the call data packet.
+    async writeCallData(data, complete, error) {
+        const callData = {
+            data,
+            complete: complete || false,
+            error: error || "",
+        };
+        await this.writePacket({
+            body: {
+                $case: 'callData',
+                callData,
+            },
+        });
+    }
+    // writePacket writes a packet to the stream.
+    async writePacket(packet) {
+        this._source.push(packet);
+    }
+    // handleMessage handles an incoming encoded Packet.
+    //
+    // note: may throw an error if the message was unexpected or invalid.
+    async handleMessage(message) {
+        return this.handlePacket(Packet.decode(message));
+    }
+    // handlePacket handles an incoming packet.
+    async handlePacket(packet) {
+        switch (packet?.body?.$case) {
+            case 'callStart':
+                return this.handleCallStart(packet.body.callStart);
+            case 'callStartResp':
+                return this.handleCallStartResp(packet.body.callStartResp);
+            case 'callData':
+                return this.handleCallData(packet.body.callData);
+        }
+    }
+    // handleCallStart handles a CallStart packet.
+    async handleCallStart(packet) {
+        // we do not implement server -> client RPCs.
+        throw new Error(`unexpected server to client rpc: ${packet.rpcService}/${packet.rpcMethod}`);
+    }
+    // handleCallStartResp handles a CallStartResp packet.
+    async handleCallStartResp(packet) {
+        if (packet.error && packet.error.length) {
+            const err = new Error(packet.error);
+            this.onStarted(err);
+            this.onComplete(err);
+        }
+    }
+    // handleCallData handles a CallData packet.
+    async handleCallData(packet) {
+        const data = packet.data;
+        if (this.dataCb && data?.length) {
+            await this.dataCb(data);
+        }
+        if (packet.error && packet.error.length) {
+            this.onComplete(new Error(packet.error));
+        }
+        else if (packet.complete) {
+            this.onComplete();
+        }
+    }
+    // close closes the active call if not already completed.
+    async close(err) {
+        if (!this.closed) {
+            await this.writeCallData(new Uint8Array(0), true, err ? err.message : "");
+        }
+        if (!err) {
+            err = new Error('call closed');
+        }
+        this.onComplete(err);
+    }
+    // _createSink initializes the sink field.
+    _createSink() {
+        return async (source) => {
+            try {
+                for await (const msg of source) {
+                    await this.handlePacket(msg);
+                }
+            }
+            catch (err) {
+                this.close(err);
+            }
+        };
+    }
+    // _createSource initializes the source field.
+    _createSource() {
+        return pushable({
+            objectMode: true,
+            onEnd: (err) => {
+                this.onComplete(err);
+            },
+        });
+    }
+}

package/dist/srpc/client.d.ts

@@ -0,0 +1,12 @@
+import { Observable } from 'rxjs';
+import type { TsProtoRpc } from './ts-proto-rpc';
+import type { OpenStreamFunc } from './stream';
+export declare class Client implements TsProtoRpc {
+    private openConnFn;
+    constructor(openConnFn: OpenStreamFunc);
+    request(service: string, method: string, data: Uint8Array): Promise<Uint8Array>;
+    clientStreamingRequest(service: string, method: string, data: Observable<Uint8Array>): Promise<Uint8Array>;
+    serverStreamingRequest(service: string, method: string, data: Uint8Array): Observable<Uint8Array>;
+    bidirectionalStreamingRequest(service: string, method: string, data: Observable<Uint8Array>): Observable<Uint8Array>;
+    private startRpc;
+}