starpc 0.0.1 → 0.1.2

package/integration/integration.ts

@@ -0,0 +1,54 @@
+import { WebSocketConn } from '../dist/srpc/websocket.js'
+import { EchoerClientImpl, EchoMsg } from '../dist/echo/echo.js'
+import WebSocket from 'isomorphic-ws'
+import { Observable } from 'rxjs'
+
+async function runRPC() {
+  const addr = 'ws://localhost:5000/demo'
+  console.log(`Connecting to ${addr}`)
+  const ws = new WebSocket(addr)
+  const channel = new WebSocketConn(ws)
+  const client = channel.buildClient()
+  const demoServiceClient = new EchoerClientImpl(client)
+
+  console.log('Calling Echo: unary call...')
+  let result = await demoServiceClient.Echo({
+    body: "Hello world!"
+  })
+  console.log('success: output', result.body)
+
+  // observable for client requests
+  const clientRequestStream = new Observable<EchoMsg>(subscriber => {
+    subscriber.next({body: 'Hello world from streaming request.'})
+    subscriber.complete()
+  })
+
+  console.log('Calling EchoClientStream: client -> server...')
+  result = await demoServiceClient.EchoClientStream(clientRequestStream)
+  console.log('success: output', result.body)
+
+  console.log('Calling EchoServerStream: server -> client...')
+  const serverStream = demoServiceClient.EchoServerStream({
+    body: 'Hello world from server to client streaming request.',
+  })
+  await new Promise<void>((resolve, reject) => {
+    serverStream.subscribe({
+      next(result) {
+        console.log('server: output', result.body)
+      },
+      complete() {
+        resolve()
+      },
+      error(err: Error) {
+        reject(err)
+      },
+    })
+  })
+}
+
+runRPC().then(() => {
+  process.exit(0)
+}).catch((err) => {
+  console.error(err)
+  process.exit(1)
+})

package/integration/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "module": "system",
+    "noEmit": false,
+    "declaration": false
+  },
+  "include": [
+    "./"
+  ]
+}

package/package.json

@@ -1,15 +1,83 @@
 {
   "name": "starpc",
-  "version": "0.0.1",
+  "version": "0.1.2",
   "description": "Streaming protobuf RPC service protocol over any two-way channel.",
-  "repository": "github:paralin/starpc",
-  "license": "Apache-2.0",
+  "license": "MIT",
   "author": {
-    "name": "Christian Stewart",
-    "email": "christian@aperture.us",
-    "url": "http://github.com/paralin"
+    "name": "Aperture Robotics LLC.",
+    "email": "support@aperture.us",
+    "url": "http://aperture.us"
   },
-  "scripts": {},
-  "prettier": {},
-  "dependencies": {}
+  "contributors": [
+    {
+      "name": "Christian Stewart",
+      "email": "christian@aperture.us",
+      "url": "http://github.com/paralin"
+    }
+  ],
+  "main": "dist/srpc/index.js",
+  "types": "./dist/srpc/index.d.ts",
+  "files": [
+    "!**/*.tsbuildinfo",
+    "Makefile",
+    "dist/echo",
+    "dist/srpc",
+    "e2e",
+    "echo",
+    "go.mod",
+    "go.sum",
+    "integration",
+    "patches",
+    "srpc"
+  ],
+  "repository": {
+    "url": "git@github.com:aperturerobotics/starpc.git"
+  },
+  "scripts": {
+    "build": "tsc --project tsconfig.build.json --outDir ./dist/",
+    "check": "tsc",
+    "codegen": "npm run gen",
+    "ci": "npm run build && npm run lint:js && npm run lint:go",
+    "format": "prettier --write './srpc/**/(*.ts|*.tsx|*.js|*.html|*.css)'",
+    "gen": "make genproto",
+    "test": "make test && npm run check",
+    "test:integration": "make integration",
+    "lint": "npm run lint:go && npm run lint:js",
+    "lint:go": "make lint",
+    "lint:js": "eslint -c .eslintrc.js --ext .ts ./{srpc,echo}/**/*.ts",
+    "postinstall": "patch-package",
+    "precommit": "npm run format"
+  },
+  "prettier": {
+    "semi": false,
+    "singleQuote": true
+  },
+  "devDependencies": {
+    "@types/varint": "^6.0.0",
+    "@typescript-eslint/eslint-plugin": "^5.27.1",
+    "@typescript-eslint/parser": "^5.27.1",
+    "esbuild": "^0.14.43",
+    "eslint": "^8.17.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-standard-with-typescript": "^21.0.1",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "husky": "^8.0.1",
+    "prettier": "^2.7.0",
+    "ts-proto": "^1.115.4",
+    "typescript": "^4.7.3"
+  },
+  "dependencies": {
+    "@libp2p/mplex": "^1.2.1",
+    "event-iterator": "^2.0.0",
+    "isomorphic-ws": "^4.0.1",
+    "it-length-prefixed": "^7.0.1",
+    "it-pipe": "^2.0.3",
+    "it-pushable": "^3.0.0",
+    "it-stream-types": "^1.0.4",
+    "it-ws": "^5.0.2",
+    "protobufjs": "^6.11.3",
+    "patch-package": "^6.4.7",
+    "rxjs": "^7.5.5",
+    "ws": "^8.8.0"
+  }
 }

package/patches/@libp2p+mplex+1.2.1.patch

@@ -0,0 +1,22 @@
+diff --git a/node_modules/@libp2p/mplex/dist/src/index.d.ts b/node_modules/@libp2p/mplex/dist/src/index.d.ts
+index 9814d45..ada275f 100644
+--- a/node_modules/@libp2p/mplex/dist/src/index.d.ts
++++ b/node_modules/@libp2p/mplex/dist/src/index.d.ts
+@@ -1,6 +1,7 @@
+ import type { Components } from '@libp2p/interfaces/components';
+ import type { StreamMuxerFactory, StreamMuxerInit } from '@libp2p/interfaces/stream-muxer';
+ import { MplexStreamMuxer } from './mplex.js';
++export { MplexStreamMuxer } from './mplex.js';
+ export interface MplexInit {
+     /**
+      * The maximum size of message that can be sent in one go in bytes.
+diff --git a/node_modules/@libp2p/mplex/dist/src/index.js b/node_modules/@libp2p/mplex/dist/src/index.js
+index 2c4cf01..4359734 100644
+--- a/node_modules/@libp2p/mplex/dist/src/index.js
++++ b/node_modules/@libp2p/mplex/dist/src/index.js
+@@ -1,4 +1,5 @@
+ import { MplexStreamMuxer } from './mplex.js';
++export { MplexStreamMuxer } from './mplex.js';
+ export class Mplex {
+     constructor(init = {}) {
+         this.protocol = '/mplex/6.7.0';

package/srpc/broadcast-channel.ts

@@ -0,0 +1,72 @@
+import type { Duplex, Sink } from 'it-stream-types'
+import { Conn } from './conn'
+import { EventIterator } from 'event-iterator'
+import { pipe } from 'it-pipe'
+
+// BroadcastChannelIterable is a AsyncIterable wrapper for BroadcastChannel.
+export class BroadcastChannelIterable<T> implements Duplex<T> {
+  // channel is the broadcast channel
+  public readonly channel: BroadcastChannel
+  // sink is the sink for incoming messages.
+  public sink: Sink<T>
+  // source is the source for outgoing messages.
+  public source: AsyncIterable<T>
+
+  constructor(channel: BroadcastChannel) {
+    this.channel = channel
+    this.sink = this._createSink()
+    this.source = this._createSource()
+  }
+
+  // _createSink initializes the sink field.
+  private _createSink(): Sink<T> {
+    return async (source) => {
+      for await (const msg of source) {
+        this.channel.postMessage(msg)
+      }
+    }
+  }
+
+  // _createSource initializes the source field.
+  private _createSource() {
+    return new EventIterator<T>((queue) => {
+      const messageListener = (ev: MessageEvent<T>) => {
+        if (ev.data) {
+          queue.push(ev.data)
+        }
+      }
+      this.channel.addEventListener('message', messageListener)
+
+      return () => {
+        this.channel.removeEventListener('message', messageListener)
+      }
+    })
+  }
+}
+
+// newBroadcastChannelIterable constructs a BroadcastChannelIterable with a channel name.
+export function newBroadcastChannelIterable<T>(
+  name: string
+): BroadcastChannelIterable<T> {
+  const channel = new BroadcastChannel(name)
+  return new BroadcastChannelIterable<T>(channel)
+}
+
+// BroadcastChannelConn implements a connection with a BroadcastChannel.
+//
+// expects Uint8Array objects over the BroadcastChannel.
+export class BroadcastChannelConn extends Conn {
+  // channel is the broadcast channel iterable
+  private channel: BroadcastChannelIterable<Uint8Array>
+
+  constructor(channel: BroadcastChannel) {
+    super()
+    this.channel = new BroadcastChannelIterable<Uint8Array>(channel)
+    pipe(this, this.channel, this)
+  }
+
+  // getBroadcastChannel returns the BroadcastChannel.
+  public getBroadcastChannel(): BroadcastChannel {
+    return this.channel.channel
+  }
+}

package/srpc/client-rpc.go

@@ -0,0 +1,163 @@
+package srpc
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+)
+
+// ClientRPC represents the client side of an on-going RPC call message stream.
+// Not concurrency safe: use a mutex if calling concurrently.
+type ClientRPC struct {
+	// ctx is the context, canceled when the rpc ends.
+	ctx context.Context
+	// ctxCancel is called when the rpc ends.
+	ctxCancel context.CancelFunc
+	// writer is the writer to write messages to
+	writer Writer
+	// service is the rpc service
+	service string
+	// method is the rpc method
+	method string
+	// dataCh contains queued data packets.
+	// closed when the client closes the channel.
+	dataCh chan []byte
+	// dataChClosed is a flag set after dataCh is closed.
+	// controlled by HandlePacket.
+	dataChClosed bool
+	// serverErr is an error set by the client.
+	// before dataCh is closed, managed by HandlePacket.
+	// immutable after dataCh is closed.
+	serverErr error
+}
+
+// NewClientRPC constructs a new ClientRPC session and writes CallStart.
+// the writer will be closed when the ClientRPC completes.
+// service and method must be specified.
+// must call Start after creating the RPC object.
+func NewClientRPC(ctx context.Context, service, method string) *ClientRPC {
+	rpc := &ClientRPC{
+		service: service,
+		method:  method,
+		dataCh:  make(chan []byte, 5),
+	}
+	rpc.ctx, rpc.ctxCancel = context.WithCancel(ctx)
+	return rpc
+}
+
+// Start sets the writer and writes the MsgSend message.
+// firstMsg can be nil, but is usually set if this is a non-streaming rpc.
+// must only be called once!
+func (r *ClientRPC) Start(writer Writer, firstMsg []byte) error {
+	select {
+	case <-r.ctx.Done():
+		r.Close()
+		return context.Canceled
+	default:
+	}
+	r.writer = writer
+	pkt := NewCallStartPacket(r.service, r.method, firstMsg)
+	if err := writer.WritePacket(pkt); err != nil {
+		r.Close()
+		return err
+	}
+	return nil
+}
+
+// ReadAll reads all returned Data packets and returns any error.
+// intended for use with unary rpcs.
+func (r *ClientRPC) ReadAll() ([][]byte, error) {
+	msgs := make([][]byte, 0, 1)
+	for {
+		select {
+		case <-r.ctx.Done():
+			return msgs, context.Canceled
+		case data, ok := <-r.dataCh:
+			if !ok {
+				return msgs, r.serverErr
+			}
+			msgs = append(msgs, data)
+		}
+	}
+}
+
+// Context is canceled when the ClientRPC is no longer valid.
+func (r *ClientRPC) Context() context.Context {
+	return r.ctx
+}
+
+// HandlePacketData handles an incoming unparsed message packet.
+// Not concurrency safe: use a mutex if calling concurrently.
+func (r *ClientRPC) HandlePacketData(data []byte) error {
+	pkt := &Packet{}
+	if err := pkt.UnmarshalVT(data); err != nil {
+		return err
+	}
+	return r.HandlePacket(pkt)
+}
+
+// HandlePacket handles an incoming parsed message packet.
+// Not concurrency safe: use a mutex if calling concurrently.
+func (r *ClientRPC) HandlePacket(msg *Packet) error {
+	if err := msg.Validate(); err != nil {
+		return err
+	}
+
+	switch b := msg.GetBody().(type) {
+	case *Packet_CallStart:
+		return r.HandleCallStart(b.CallStart)
+	case *Packet_CallData:
+		return r.HandleCallData(b.CallData)
+	case *Packet_CallStartResp:
+		return r.HandleCallStartResp(b.CallStartResp)
+	default:
+		return nil
+	}
+}
+
+// HandleCallStart handles the call start packet.
+func (r *ClientRPC) HandleCallStart(pkt *CallStart) error {
+	// server-to-client calls not supported
+	return errors.Wrap(ErrUnrecognizedPacket, "call start packet unexpected")
+}
+
+// HandleCallData handles the call data packet.
+func (r *ClientRPC) HandleCallData(pkt *CallData) error {
+	if r.dataChClosed {
+		return ErrCompleted
+	}
+
+	if data := pkt.GetData(); len(data) != 0 {
+		select {
+		case <-r.ctx.Done():
+			return context.Canceled
+		case r.dataCh <- data:
+		}
+	}
+
+	complete := pkt.GetComplete()
+	if err := pkt.GetError(); len(err) != 0 {
+		complete = true
+		r.serverErr = errors.New(err)
+	}
+
+	if complete {
+		r.dataChClosed = true
+		close(r.dataCh)
+	}
+
+	return nil
+}
+
+// HandleCallStartResp handles the CallStartResp packet.
+func (r *ClientRPC) HandleCallStartResp(resp *CallStartResp) error {
+	// client-side calls not supported
+	return errors.Wrap(ErrUnrecognizedPacket, "call start resp packet unexpected")
+}
+
+// Close releases any resources held by the ClientRPC.
+// not concurrency safe with HandlePacket.
+func (r *ClientRPC) Close() {
+	r.ctxCancel()
+	_ = r.writer.Close()
+}

package/srpc/client-rpc.ts

@@ -0,0 +1,197 @@
+import type { CallData, CallStart, CallStartResp } from './rpcproto'
+import { Packet } from './rpcproto'
+import type { Sink } from 'it-stream-types'
+import { pushable } from 'it-pushable'
+
+// DataCb is a callback to handle incoming RPC messages.
+// Returns true if more data is expected, false otherwise.
+// If returns undefined, assumes more data is expected.
+export type DataCb = (data: Uint8Array) => Promise<boolean | void>
+
+// ClientRPC is an ongoing RPC from the client side.
+export class ClientRPC {
+  // sink is the data sink for incoming messages.
+  public sink: Sink<Packet>
+  // source is the packet source for outgoing Packets.
+  public source: AsyncIterable<Packet>
+  // _source is used to write to the source.
+  private readonly _source: {
+    push: (val: Packet) => void
+    end: (err?: Error) => void
+  }
+  // service is the rpc service
+  private service: string
+  // method is the rpc method
+  private method: string
+  // dataCb is called with any incoming data.
+  private dataCb?: DataCb
+  // started is resolved when the request starts.
+  private started: Promise<void>
+  // onStarted is called by the message handler when the request starts.
+  private onStarted?: (err?: Error) => void
+  // complete is resolved when the request completes.
+  // rejected with an error if the call encountered any error.
+  private complete: Promise<void>
+  // onComplete is called by the message handler when the call completes.
+  private onComplete?: (err?: Error) => void
+  // closed indicates close has been called
+  private closed: boolean
+
+  constructor(service: string, method: string, dataCb: DataCb | null) {
+    this.closed = false
+    this.sink = this._createSink()
+    const sourcev = this._createSource()
+    this.source = sourcev
+    this._source = sourcev
+    this.service = service
+    this.method = method
+    if (dataCb) {
+      this.dataCb = dataCb
+    }
+    this.started = new Promise<void>((resolveStarted, rejectStarted) => {
+      this.onStarted = (err?: Error) => {
+        if (err) {
+          rejectStarted(err)
+        } else {
+          resolveStarted()
+        }
+      }
+    })
+    this.complete = new Promise<void>((resolveComplete, rejectComplete) => {
+      this.onComplete = (err?: Error) => {
+        this.closed = true
+        if (err) {
+          rejectComplete(err)
+        } else {
+          resolveComplete()
+        }
+      }
+    })
+  }
+
+  // waitStarted returns the started promise.
+  public waitStarted(): Promise<void> {
+    return this.started
+  }
+
+  // waitComplete returns the complete promise.
+  public waitComplete(): Promise<void> {
+    return this.complete
+  }
+
+  // writeCallStart writes the call start packet.
+  public async writeCallStart(data?: Uint8Array) {
+    const callStart: CallStart = {
+      rpcService: this.service,
+      rpcMethod: this.method,
+      data: data || new Uint8Array(0),
+    }
+    await this.writePacket({
+      body: {
+        $case: 'callStart',
+        callStart,
+      },
+    })
+  }
+
+  // writeCallData writes the call data packet.
+  public async writeCallData(data: Uint8Array, complete?: boolean, error?: string) {
+    const callData: CallData = {
+      data,
+      complete: complete || false,
+      error: error || "",
+    }
+    await this.writePacket({
+      body: {
+        $case: 'callData',
+        callData,
+      },
+    })
+  }
+
+  // writePacket writes a packet to the stream.
+  private async writePacket(packet: Packet) {
+    this._source.push(packet)
+  }
+
+  // handleMessage handles an incoming encoded Packet.
+  //
+  // note: may throw an error if the message was unexpected or invalid.
+  public async handleMessage(message: Uint8Array) {
+    return this.handlePacket(Packet.decode(message))
+  }
+
+  // handlePacket handles an incoming packet.
+  public async handlePacket(packet: Partial<Packet>) {
+    switch (packet?.body?.$case) {
+      case 'callStart':
+        return this.handleCallStart(packet.body.callStart)
+      case 'callStartResp':
+        return this.handleCallStartResp(packet.body.callStartResp)
+      case 'callData':
+        return this.handleCallData(packet.body.callData)
+    }
+  }
+
+  // handleCallStart handles a CallStart packet.
+  public async handleCallStart(packet: Partial<CallStart>) {
+    // we do not implement server -> client RPCs.
+    throw new Error(`unexpected server to client rpc: ${packet.rpcService}/${packet.rpcMethod}`)
+  }
+
+  // handleCallStartResp handles a CallStartResp packet.
+  public async handleCallStartResp(packet: Partial<CallStartResp>) {
+    if (packet.error && packet.error.length) {
+      const err = new Error(packet.error)
+      this.onStarted!(err)
+      this.onComplete!(err)
+    }
+  }
+
+  // handleCallData handles a CallData packet.
+  public async handleCallData(packet: Partial<CallData>) {
+    const data = packet.data
+    if (this.dataCb && data?.length) {
+      await this.dataCb(data)
+    }
+    if (packet.error && packet.error.length) {
+      this.onComplete!(new Error(packet.error))
+    } else if (packet.complete) {
+      this.onComplete!()
+    }
+  }
+
+  // close closes the active call if not already completed.
+  public async close(err?: Error) {
+    if (!this.closed) {
+      await this.writeCallData(new Uint8Array(0), true, err ? err.message : "")
+    }
+    if (!err) {
+      err = new Error('call closed')
+    }
+    this.onComplete!(err)
+  }
+
+  // _createSink initializes the sink field.
+  private _createSink(): Sink<Packet> {
+    return async (source) => {
+      try {
+        for await (const msg of source) {
+          await this.handlePacket(msg)
+        }
+      } catch (err) {
+        this.close(err as Error)
+      }
+    }
+  }
+
+  // _createSource initializes the source field.
+  private _createSource() {
+    return pushable<Packet>({
+      objectMode: true,
+      onEnd: (err?: Error): void => {
+        this.onComplete!(err)
+      },
+    })
+  }
+}