starpc 0.0.1 → 0.1.2

package/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2022 Aperture Robotics, LLC.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/Makefile

@@ -0,0 +1,140 @@
+PROTOWRAP=hack/bin/protowrap
+PROTOC_GEN_GO=hack/bin/protoc-gen-go
+PROTOC_GEN_VTPROTO=hack/bin/protoc-gen-go-vtproto
+PROTOC_GEN_STARPC=hack/bin/protoc-gen-go-starpc
+GOIMPORTS=hack/bin/goimports
+GOLANGCI_LINT=hack/bin/golangci-lint
+GO_MOD_OUTDATED=hack/bin/go-mod-outdated
+export GO111MODULE=on
+GOLIST=go list -f "{{ .Dir }}" -m
+
+all:
+
+vendor:
+	go mod vendor
+
+$(PROTOC_GEN_GO):
+	cd ./hack; \
+	go build -v \
+		-o ./bin/protoc-gen-go \
+		github.com/golang/protobuf/protoc-gen-go
+
+$(PROTOC_GEN_VTPROTO):
+	cd ./hack; \
+	go build -v \
+		-o ./bin/protoc-gen-go-vtproto \
+		github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto
+
+$(PROTOC_GEN_STARPC):
+	cd ./hack; \
+	go build -v \
+		-o ./bin/protoc-gen-go-starpc \
+		github.com/aperturerobotics/starpc/cmd/protoc-gen-go-starpc
+
+$(GOIMPORTS):
+	cd ./hack; \
+	go build -v \
+		-o ./bin/goimports \
+		golang.org/x/tools/cmd/goimports
+
+$(PROTOWRAP):
+	cd ./hack; \
+	go build -v \
+		-o ./bin/protowrap \
+		github.com/square/goprotowrap/cmd/protowrap
+
+$(GOLANGCI_LINT):
+	cd ./hack; \
+	go build -v \
+		-o ./bin/golangci-lint \
+		github.com/golangci/golangci-lint/cmd/golangci-lint
+
+$(GO_MOD_OUTDATED):
+	cd ./hack; \
+	go build -v \
+		-o ./bin/go-mod-outdated \
+		github.com/psampaz/go-mod-outdated
+
+.PHONY: gengo
+gengo: $(GOIMPORTS) $(PROTOWRAP) $(PROTOC_GEN_GO) $(PROTOC_GEN_VTPROTO) $(PROTOC_GEN_STARPC)
+	go mod vendor
+	shopt -s globstar; \
+	set -eo pipefail; \
+	export PROJECT=$$(go list -m); \
+	export PATH=$$(pwd)/hack/bin:$${PATH}; \
+	mkdir -p $$(pwd)/vendor/$$(dirname $${PROJECT}); \
+	rm $$(pwd)/vendor/$${PROJECT} || true; \
+	ln -s $$(pwd) $$(pwd)/vendor/$${PROJECT} ; \
+	$(PROTOWRAP) \
+		-I $$(pwd)/vendor \
+		--go_out=$$(pwd)/vendor \
+		--go-starpc_out=$$(pwd)/vendor \
+		--go-vtproto_out=$$(pwd)/vendor \
+		--go-vtproto_opt=features=marshal+unmarshal+size+equal \
+		--proto_path $$(pwd)/vendor \
+		--print_structure \
+		--only_specified_files \
+		$$(\
+			git \
+				ls-files "*.proto" |\
+				xargs printf -- \
+				"$$(pwd)/vendor/$${PROJECT}/%s "); \
+	rm $$(pwd)/vendor/$${PROJECT} || true
+	$(GOIMPORTS) -w ./
+
+node_modules:
+	yarn install
+
+.PHONY: gents
+gents: $(PROTOWRAP) node_modules
+	go mod vendor
+	shopt -s globstar; \
+	set -eo pipefail; \
+	export PROJECT=$$(go list -m); \
+	export PATH=$$(pwd)/hack/bin:$${PATH}; \
+	mkdir -p $$(pwd)/vendor/$$(dirname $${PROJECT}); \
+	rm $$(pwd)/vendor/$${PROJECT} || true; \
+	ln -s $$(pwd) $$(pwd)/vendor/$${PROJECT} ; \
+	$(PROTOWRAP) \
+		-I $$(pwd)/vendor \
+		--plugin=./node_modules/.bin/protoc-gen-ts_proto \
+		--ts_proto_out=$$(pwd)/vendor \
+		--ts_proto_opt=forceLong=long \
+		--ts_proto_opt=oneof=unions \
+		--ts_proto_opt=esModuleInterop=true \
+		--proto_path $$(pwd)/vendor \
+		--print_structure \
+		--only_specified_files \
+		$$(\
+			git \
+				ls-files "*.proto" |\
+				xargs printf -- \
+				"$$(pwd)/vendor/$${PROJECT}/%s "); \
+	go mod vendor
+
+.PHONY: genproto
+genproto: gengo gents
+
+.PHONY: gen
+gen: genproto
+
+outdated: $(GO_MOD_OUTDATED)
+	go list -mod=mod -u -m -json all | $(GO_MOD_OUTDATED) -update -direct
+
+list: $(GO_MOD_OUTDATED)
+	go list -mod=mod -u -m -json all | $(GO_MOD_OUTDATED)
+
+lint: $(GOLANGCI_LINT)
+	$(GOLANGCI_LINT) run
+
+fix: $(GOLANGCI_LINT)
+	$(GOLANGCI_LINT) run --fix
+
+.PHONY: test
+test:
+	go test -v ./...
+
+.PHONY: integration
+integration: node_modules vendor
+	cd ./integration && \
+		bash ./integration.bash

package/README.md

@@ -1,56 +1,158 @@
-# Stream RPC (starpc)
+# Stream RPC
 
-**starpc** is a Protobuf RPC implementation leveraging an external Stream
-multiplexer and Message framing system like HTTP/2 or Yamux.
+**starpc** is a fully-featured client and server for [Proto3 services] in both
+TypeScript and Go.
 
-Uses any two-way ordered message channel like WebSocket or BroadcastChannel.
+[Proto3 services]: https://developers.google.com/protocol-buffers/docs/proto3#services
 
-Implementation features:
+One of the first libraries to support **client-to-server streaming** RPCs.
 
- - Generates a Go server for proto3 services that does not use reflection.
- - Uses the **ts-proto** RPC interface to implement TypeScript clients.
- - Each RPC call is mapped to a Stream preventing head-of-line blocking.
+The [rpcproto](./srpc/rpcproto.proto) file contains the entire protocol.
 
-The Go backend provides an interface for forwarding the incoming RPC calls to
-the appropriate backend service.
+Leverages the Stream multiplexing of the underlying transport; for example:
+HTTP/2 or [libp2p-mplex] over a WebSocket.
+
+[libp2p-mplex]: https://github.com/libp2p/js-libp2p-mplex
+
+The server side has not yet been implemented in TypeScript.
 
 # Usage
 
-## Go
+Starting with the [protobuf-project] repository on the "starpc" branch.
+
+Use "git add" to add your new .proto files, then `yarn gen` to generate the
+TypeScript and Go code for them.
+
+# Examples
+
+See the [protobuf-project] template on the "starpc" branch.
+
+[protobuf-project]: https://github.com/aperturerobotics/protobuf-project
+
+## Protobuf
 
-TODO
+The following examples use the [echo](./echo/echo.proto) protobuf sample.
 
-Copy the Go project boilerplate files:
+```protobuf
+syntax = "proto3";
+package echo;
 
- - Makefile: implements "make gengo" to generate Go code with protoc.
- - hack/: downloads & builds the necessary protoc-gen-go tools.
+// Echoer service returns the given message.
+service Echoer {
+  // Echo returns the given message.
+  rpc Echo(EchoMsg) returns (EchoMsg);
+  // EchoServerStream is an example of a server -> client one-way stream.
+  rpc EchoServerStream(EchoMsg) returns (stream EchoMsg);
+  // EchoClientStream is an example of client->server one-way stream.
+  rpc EchoClientStream(stream EchoMsg) returns (EchoMsg);
+  // EchoBidiStream is an example of a two-way stream.
+  rpc EchoBidiStream(stream EchoMsg) returns (stream EchoMsg);
+}
 
-To generate code for .proto files in your repository:
+// EchoMsg is the message body for Echo.
+message EchoMsg {
+  string body = 1;
+}
+```
+
+## Go
 
- - `git add .` - stage all changes first.
- - `make gengo` - generate the protobuf files.
+A basic example can be found in the [e2e test]:
+
+```go
+// construct the server
+echoServer := &echo.EchoServer{}
+mux := srpc.NewMux()
+if err := echo.SRPCRegisterEchoer(mux, echoServer); err != nil {
+	t.Fatal(err.Error())
+}
+server := srpc.NewServer(mux)
+
+// create an in-memory connection to the server
+openStream := srpc.NewServerPipe(server)
+
+// construct the client
+client := srpc.NewClient(openStream)
+
+// construct the client rpc interface
+clientEcho := echo.NewSRPCEchoerClient(client)
+ctx := context.Background()
+bodyTxt := "hello world"
+out, err := clientEcho.Echo(ctx, &echo.EchoMsg{
+	Body: bodyTxt,
+})
+if err != nil {
+	t.Fatal(err.Error())
+}
+if out.GetBody() != bodyTxt {
+	t.Fatalf("expected %q got %q", bodyTxt, out.GetBody())
+}
+```
 
-Example of creating & serving a service in Go: (TODO)
+[e2e test]: ./e2e/e2e_test.go
 
 ## TypeScript
 
 See the ts-proto README to generate the TypeScript for your protobufs.
 
+Also check out the [integration](./integration/integration.ts) test.
+
+Supports any AsyncIterable communication channel with an included implementation
+for WebSockets.
+
+This repository uses protowrap, see the [Makefile](./Makefile).
+
 ```typescript
-import { RpcClient, WebSocketConn } from 'starpc'
-import { DemoServiceClientImpl } from './demo'
+import { WebSocketConn } from '../srpc'
+import { EchoerClientImpl } from '../echo/echo'
 
 const ws = new WebSocket('ws://localhost:5000/demo')
 const channel = new WebSocketConn(ws)
-const rpc = new RpcClient(channel)
-const demoServiceClient = new DemoServiceClientImpl(rpc)
+const client = channel.buildClient()
+const demoServiceClient = new EchoerClientImpl(client)
 
-const result = await demoServiceClient.DemoEcho({
-  msg: "Hello world!"
+const result = await demoServiceClient.Echo({
+  body: "Hello world!"
 })
-console.log('output', result.msg)
+console.log('output', result.body)
+
+const clientRequestStream = new Observable<EchoMsg>(subscriber => {
+  subscriber.next({body: 'Hello world from streaming request.'})
+  subscriber.complete()
+})
+
+console.log('Calling EchoClientStream: client -> server...')
+result = await demoServiceClient.EchoClientStream(clientRequestStream)
+console.log('success: output', result.body)
 ```
 
 `WebSocketConn` uses [js-libp2p-mplex] to multiplex streams over the WebSocket.
 
 [js-libp2p-mplex]: https://github.com/libp2p/js-libp2p-mplex
+
+# Attribution
+
+`protoc-gen-go-starpc` is a heavily modified version of `protoc-gen-go-drpc`.
+
+Be sure to check out [drpc] as well: it's compatible with grpc, twirp, and more.
+
+[drpc]: https://github.com/storj/drpc
+
+Uses [vtprotobuf] to generate Protobuf marshal / unmarshal code.
+
+[vtprotobuf]: https://github.com/planetscale/vtprotobuf
+
+# Support
+
+Starpc is built & supported by Aperture Robotics, LLC.
+
+Community contributions and discussion are welcomed!
+
+Please open a [GitHub issue] with any questions / issues.
+
+[GitHub issue]: https://github.com/aperturerobotics/bifrost/issues/new
+
+... or feel free to reach out on [Matrix Chat] or [Discord].
+
+[Discord]: https://discord.gg/KJutMESRsT
+[Matrix Chat]: https://matrix.to/#/#aperturerobotics:matrix.org

package/dist/echo/echo.d.ts

@@ -0,0 +1,59 @@
+import Long from 'long';
+import * as _m0 from 'protobufjs/minimal';
+import { Observable } from 'rxjs';
+export declare const protobufPackage = "echo";
+/** EchoMsg is the message body for Echo. */
+export interface EchoMsg {
+    body: string;
+}
+export declare const EchoMsg: {
+    encode(message: EchoMsg, writer?: _m0.Writer): _m0.Writer;
+    decode(input: _m0.Reader | Uint8Array, length?: number): EchoMsg;
+    fromJSON(object: any): EchoMsg;
+    toJSON(message: EchoMsg): unknown;
+    fromPartial<I extends {
+        body?: string | undefined;
+    } & {
+        body?: string | undefined;
+    } & Record<Exclude<keyof I, "body">, never>>(object: I): EchoMsg;
+};
+/** Echoer service returns the given message. */
+export interface Echoer {
+    /** Echo returns the given message. */
+    Echo(request: EchoMsg): Promise<EchoMsg>;
+    /** EchoServerStream is an example of a server -> client one-way stream. */
+    EchoServerStream(request: EchoMsg): Observable<EchoMsg>;
+    /** EchoClientStream is an example of client->server one-way stream. */
+    EchoClientStream(request: Observable<EchoMsg>): Promise<EchoMsg>;
+    /** EchoBidiStream is an example of a two-way stream. */
+    EchoBidiStream(request: Observable<EchoMsg>): Observable<EchoMsg>;
+}
+export declare class EchoerClientImpl implements Echoer {
+    private readonly rpc;
+    constructor(rpc: Rpc);
+    Echo(request: EchoMsg): Promise<EchoMsg>;
+    EchoServerStream(request: EchoMsg): Observable<EchoMsg>;
+    EchoClientStream(request: Observable<EchoMsg>): Promise<EchoMsg>;
+    EchoBidiStream(request: Observable<EchoMsg>): Observable<EchoMsg>;
+}
+interface Rpc {
+    request(service: string, method: string, data: Uint8Array): Promise<Uint8Array>;
+    clientStreamingRequest(service: string, method: string, data: Observable<Uint8Array>): Promise<Uint8Array>;
+    serverStreamingRequest(service: string, method: string, data: Uint8Array): Observable<Uint8Array>;
+    bidirectionalStreamingRequest(service: string, method: string, data: Observable<Uint8Array>): Observable<Uint8Array>;
+}
+declare type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
+export declare type DeepPartial<T> = T extends Builtin ? T : T extends Long ? string | number | Long : T extends Array<infer U> ? Array<DeepPartial<U>> : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>> : T extends {
+    $case: string;
+} ? {
+    [K in keyof Omit<T, '$case'>]?: DeepPartial<T[K]>;
+} & {
+    $case: T['$case'];
+} : T extends {} ? {
+    [K in keyof T]?: DeepPartial<T[K]>;
+} : Partial<T>;
+declare type KeysOfUnion<T> = T extends T ? keyof T : never;
+export declare type Exact<P, I extends P> = P extends Builtin ? P : P & {
+    [K in keyof P]: Exact<P[K], I[K]>;
+} & Record<Exclude<keyof I, KeysOfUnion<P>>, never>;
+export {};

package/dist/echo/echo.js

@@ -0,0 +1,85 @@
+/* eslint-disable */
+import Long from 'long';
+import * as _m0 from 'protobufjs/minimal';
+import { map } from 'rxjs/operators';
+export const protobufPackage = 'echo';
+function createBaseEchoMsg() {
+    return { body: '' };
+}
+export const EchoMsg = {
+    encode(message, writer = _m0.Writer.create()) {
+        if (message.body !== '') {
+            writer.uint32(10).string(message.body);
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseEchoMsg();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1:
+                    message.body = reader.string();
+                    break;
+                default:
+                    reader.skipType(tag & 7);
+                    break;
+            }
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            body: isSet(object.body) ? String(object.body) : '',
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        message.body !== undefined && (obj.body = message.body);
+        return obj;
+    },
+    fromPartial(object) {
+        const message = createBaseEchoMsg();
+        message.body = object.body ?? '';
+        return message;
+    },
+};
+export class EchoerClientImpl {
+    rpc;
+    constructor(rpc) {
+        this.rpc = rpc;
+        this.Echo = this.Echo.bind(this);
+        this.EchoServerStream = this.EchoServerStream.bind(this);
+        this.EchoClientStream = this.EchoClientStream.bind(this);
+        this.EchoBidiStream = this.EchoBidiStream.bind(this);
+    }
+    Echo(request) {
+        const data = EchoMsg.encode(request).finish();
+        const promise = this.rpc.request('echo.Echoer', 'Echo', data);
+        return promise.then((data) => EchoMsg.decode(new _m0.Reader(data)));
+    }
+    EchoServerStream(request) {
+        const data = EchoMsg.encode(request).finish();
+        const result = this.rpc.serverStreamingRequest('echo.Echoer', 'EchoServerStream', data);
+        return result.pipe(map((data) => EchoMsg.decode(new _m0.Reader(data))));
+    }
+    EchoClientStream(request) {
+        const data = request.pipe(map((request) => EchoMsg.encode(request).finish()));
+        const promise = this.rpc.clientStreamingRequest('echo.Echoer', 'EchoClientStream', data);
+        return promise.then((data) => EchoMsg.decode(new _m0.Reader(data)));
+    }
+    EchoBidiStream(request) {
+        const data = request.pipe(map((request) => EchoMsg.encode(request).finish()));
+        const result = this.rpc.bidirectionalStreamingRequest('echo.Echoer', 'EchoBidiStream', data);
+        return result.pipe(map((data) => EchoMsg.decode(new _m0.Reader(data))));
+    }
+}
+if (_m0.util.Long !== Long) {
+    _m0.util.Long = Long;
+    _m0.configure();
+}
+function isSet(value) {
+    return value !== null && value !== undefined;
+}

package/dist/srpc/broadcast-channel.d.ts

@@ -0,0 +1,16 @@
+import type { Duplex, Sink } from 'it-stream-types';
+import { Conn } from './conn';
+export declare class BroadcastChannelIterable<T> implements Duplex<T> {
+    readonly channel: BroadcastChannel;
+    sink: Sink<T>;
+    source: AsyncIterable<T>;
+    constructor(channel: BroadcastChannel);
+    private _createSink;
+    private _createSource;
+}
+export declare function newBroadcastChannelIterable<T>(name: string): BroadcastChannelIterable<T>;
+export declare class BroadcastChannelConn extends Conn {
+    private channel;
+    constructor(channel: BroadcastChannel);
+    getBroadcastChannel(): BroadcastChannel;
+}

package/dist/srpc/broadcast-channel.js

@@ -0,0 +1,60 @@
+import { Conn } from './conn';
+import { EventIterator } from 'event-iterator';
+import { pipe } from 'it-pipe';
+// BroadcastChannelIterable is a AsyncIterable wrapper for BroadcastChannel.
+export class BroadcastChannelIterable {
+    // channel is the broadcast channel
+    channel;
+    // sink is the sink for incoming messages.
+    sink;
+    // source is the source for outgoing messages.
+    source;
+    constructor(channel) {
+        this.channel = channel;
+        this.sink = this._createSink();
+        this.source = this._createSource();
+    }
+    // _createSink initializes the sink field.
+    _createSink() {
+        return async (source) => {
+            for await (const msg of source) {
+                this.channel.postMessage(msg);
+            }
+        };
+    }
+    // _createSource initializes the source field.
+    _createSource() {
+        return new EventIterator((queue) => {
+            const messageListener = (ev) => {
+                if (ev.data) {
+                    queue.push(ev.data);
+                }
+            };
+            this.channel.addEventListener('message', messageListener);
+            return () => {
+                this.channel.removeEventListener('message', messageListener);
+            };
+        });
+    }
+}
+// newBroadcastChannelIterable constructs a BroadcastChannelIterable with a channel name.
+export function newBroadcastChannelIterable(name) {
+    const channel = new BroadcastChannel(name);
+    return new BroadcastChannelIterable(channel);
+}
+// BroadcastChannelConn implements a connection with a BroadcastChannel.
+//
+// expects Uint8Array objects over the BroadcastChannel.
+export class BroadcastChannelConn extends Conn {
+    // channel is the broadcast channel iterable
+    channel;
+    constructor(channel) {
+        super();
+        this.channel = new BroadcastChannelIterable(channel);
+        pipe(this, this.channel, this);
+    }
+    // getBroadcastChannel returns the BroadcastChannel.
+    getBroadcastChannel() {
+        return this.channel.channel;
+    }
+}

package/dist/srpc/client-rpc.d.ts

@@ -0,0 +1,31 @@
+import type { CallData, CallStart, CallStartResp } from './rpcproto';
+import { Packet } from './rpcproto';
+import type { Sink } from 'it-stream-types';
+export declare type DataCb = (data: Uint8Array) => Promise<boolean | void>;
+export declare class ClientRPC {
+    sink: Sink<Packet>;
+    source: AsyncIterable<Packet>;
+    private readonly _source;
+    private service;
+    private method;
+    private dataCb?;
+    private started;
+    private onStarted?;
+    private complete;
+    private onComplete?;
+    private closed;
+    constructor(service: string, method: string, dataCb: DataCb | null);
+    waitStarted(): Promise<void>;
+    waitComplete(): Promise<void>;
+    writeCallStart(data?: Uint8Array): Promise<void>;
+    writeCallData(data: Uint8Array, complete?: boolean, error?: string): Promise<void>;
+    private writePacket;
+    handleMessage(message: Uint8Array): Promise<void>;
+    handlePacket(packet: Partial<Packet>): Promise<void>;
+    handleCallStart(packet: Partial<CallStart>): Promise<void>;
+    handleCallStartResp(packet: Partial<CallStartResp>): Promise<void>;
+    handleCallData(packet: Partial<CallData>): Promise<void>;
+    close(err?: Error): Promise<void>;
+    private _createSink;
+    private _createSource;
+}