ts-proto 1.117.0 → 1.119.0

package/CHANGELOG.md

@@ -1,3 +1,24 @@
+# [1.119.0](https://github.com/stephenh/ts-proto/compare/v1.118.0...v1.119.0) (2022-07-21)
+
+
+### Features
+
+* support grpc-web client straming ([#617](https://github.com/stephenh/ts-proto/issues/617)) ([d3e7f1f](https://github.com/stephenh/ts-proto/commit/d3e7f1f4aac12db87c54d6357e557a351c96a2ca))
+
+# [1.118.0](https://github.com/stephenh/ts-proto/compare/v1.117.1...v1.118.0) (2022-07-19)
+
+
+### Features
+
+* add dynamic upStreamCodes option ([#618](https://github.com/stephenh/ts-proto/issues/618)) ([3091023](https://github.com/stephenh/ts-proto/commit/309102313273e09aef3a8480f4b46360ad82adaa))
+
+## [1.117.1](https://github.com/stephenh/ts-proto/compare/v1.117.0...v1.117.1) (2022-07-16)
+
+
+### Bug Fixes
+
+* import protobufjs/minimal with importSuffix ([#616](https://github.com/stephenh/ts-proto/issues/616)) ([b86291c](https://github.com/stephenh/ts-proto/commit/b86291c58b1cb9d4064944a6371e928b564aef7f))
+
 # [1.117.0](https://github.com/stephenh/ts-proto/compare/v1.116.1...v1.117.0) (2022-07-05)
 
 

package/README.markdown

@@ -99,6 +99,13 @@ plugins:
     path: ../node_modules/ts-proto/protoc-gen-ts_proto
 ```
 
+To prevent `buf push` from reading irrelevent `.proto` files, configure `buf.yaml` like so:
+
+```yaml
+build:
+  excludes: [node_modules]
+```
+
 You can also use the official plugin published to the Buf Registry.
 
 ```yaml
@@ -110,6 +117,8 @@ plugins:
 
 # Goals
 
+In terms of the code that `ts-proto` generates, the general goals are:
+
 - Idiomatic TypeScript/ES6 types
   - `ts-proto` is a clean break from either the built-in Google/Java-esque JS code of `protoc` or the "make `.d.ts` files the `*.js` comments" approach of `protobufjs`
   - (Techically the `protobufjs/minimal` package is used for actually reading/writing bytes.)
@@ -118,6 +127,17 @@ plugins:
   - As much as possible, types are just interfaces, so you can work with messages just like regular hashes/data structures.
 - Only supports codegen `*.proto`-to-`*.ts` workflow, currently no runtime reflection/loading of dynamic `.proto` files
 
+## Non-Goals
+
+Note that ts-proto is not an out-of-the-box RPC framework; instead it's more of a swiss-army knife (as witnessed by its many config options), that lets you build _exactly_ the RPC framework you'd like on top of it (i.e. that best integrates with your company's protobuf ecosystem; for better or worse, protobuf RPC is still a somewhat fragmented ecosystem).
+
+If you'd like an out-of-the-box RPC framework built on top of ts-proto, there are a few examples:
+
+- [nice-grpc](https://github.com/deeplay-io/nice-grpc)
+- [starpc](https://github.com/aperturerobotics/starpc)
+
+(Note for potential contributors, if you develop other frameworks/mini-frameworks, or even blog posts/tutorials, on using `ts-proto`, we're happy to link to them.)
+
 # Example Types
 
 The generated types are "just data", i.e.:
@@ -483,6 +503,8 @@ If you need ts-proto customizations or priority support for your company, you ca
 
 # Development
 
+This section describes how to contribute directly to ts-proto, i.e. it's not required for running `ts-proto` in `protoc` or using the generated TypeScript.
+
 **Requirements**
 
 - [Docker](https://www.docker.com) or [protoc](https://github.com/protocolbuffers/protobuf/releases) v3.19.1

package/build/generate-grpc-web.js

@@ -18,7 +18,7 @@ function generateGrpcClientImpl(ctx, _fileDesc, serviceDesc) {
     // Create the constructor(rpc: Rpc)
     chunks.push((0, ts_poet_1.code) `
     private readonly rpc: Rpc;
-    
+
     constructor(rpc: Rpc) {
   `);
     chunks.push((0, ts_poet_1.code) `this.rpc = rpc;`);
@@ -46,9 +46,11 @@ function generateRpcMethod(ctx, serviceDesc, methodDesc) {
         return (0, ts_poet_1.code) `
     ${methodDesc.formattedName}(
       request: ${inputType},
-      metadata?: grpc.Metadata,
-    ): ${returns} {
-      throw new Error('ts-proto does not yet support client streaming!');
+      options?: {
+        metadata?: grpc.Metadata,
+        rpcOptions?: grpc.RpcOptions,
+    }): ${returns} {
+      return this.rpc.stream(${methodDescName(serviceDesc, methodDesc)}, request, options?.metadata, options?.rpcOptions)
     }
   `;
     }
@@ -83,7 +85,7 @@ exports.generateGrpcServiceDesc = generateGrpcServiceDesc;
  * implementation.
  */
 function generateGrpcMethodDesc(ctx, serviceDesc, methodDesc) {
-    const inputType = (0, types_1.requestType)(ctx, methodDesc);
+    const inputType = (0, types_1.rawRequestType)(ctx, methodDesc);
     const outputType = (0, types_1.responseType)(ctx, methodDesc);
     // grpc-web expects this to be a class, but the ts-proto messages are just interfaces.
     //
@@ -109,13 +111,15 @@ function generateGrpcMethodDesc(ctx, serviceDesc, methodDesc) {
     deserializeBinary(data: Uint8Array) {
       return { ...${outputType}.decode(data), toObject() { return this; } };
     }
-  }`;
+}`;
+    const streaming = methodDesc.clientStreaming || methodDesc.serverStreaming;
+    const methodDef = streaming ? 'MethodDefinitionish' : 'UnaryMethodDefinitionish';
     return (0, ts_poet_1.code) `
-    export const ${methodDescName(serviceDesc, methodDesc)}: UnaryMethodDefinitionish = {
+    export const ${methodDescName(serviceDesc, methodDesc)}: ${methodDef} = {
       methodName: "${methodDesc.name}",
       service: ${serviceDesc.name}Desc,
-      requestStream: false,
-      responseStream: ${methodDesc.serverStreaming ? 'true' : 'false'},
+      requestStream: ${methodDesc.clientStreaming ? true : false},
+      responseStream: ${methodDesc.serverStreaming ? true : false},
       requestType: ${requestFn} as any,
       responseType: ${responseFn} as any,
     };
@@ -133,6 +137,10 @@ function addGrpcWebMisc(ctx, hasStreamingMethods) {
     interface UnaryMethodDefinitionishR extends ${grpc}.UnaryMethodDefinition<any, any> { requestStream: any; responseStream: any; }
   `);
     chunks.push((0, ts_poet_1.code) `type UnaryMethodDefinitionish = UnaryMethodDefinitionishR;`);
+    chunks.push((0, ts_poet_1.code) `
+    interface MethodDefinitionishR extends ${grpc}.MethodDefinition<any, any> { requestStream: any; responseStream: any; }
+`);
+    chunks.push((0, ts_poet_1.code) `type MethodDefinitionish = MethodDefinitionishR;`);
     chunks.push(generateGrpcWebRpcType(ctx, options.returnObservable, hasStreamingMethods));
     chunks.push(generateGrpcWebImpl(ctx, options.returnObservable, hasStreamingMethods));
     return (0, ts_poet_1.joinCode)(chunks, { on: '\n\n' });
@@ -156,8 +164,14 @@ function generateGrpcWebRpcType(ctx, returnObservable, hasStreamingMethods) {
         methodDesc: T,
         request: any,
         metadata: grpc.Metadata | undefined,
-      ): ${(0, types_1.observableType)(ctx)}<any>;
-    `);
+      ): ${(0, types_1.observableType)(ctx)}<any>;`);
+        chunks.push((0, ts_poet_1.code) `
+  stream<T extends MethodDefinitionish>(
+    methodDesc: T,
+    request: ${(0, types_1.observableType)(ctx)}<any>,
+    metadata: grpc.Metadata | undefined,
+    rpcOptions: grpc.RpcOptions | undefined
+  ): ${(0, types_1.observableType)(ctx)}<any>;`);
     }
     chunks.push((0, ts_poet_1.code) `}`);
     return (0, ts_poet_1.joinCode)(chunks, { on: '\n' });
@@ -170,6 +184,7 @@ function generateGrpcWebImpl(ctx, returnObservable, hasStreamingMethods) {
       ${hasStreamingMethods ? 'streamingTransport?: grpc.TransportFactory,' : ``}
       debug?: boolean,
       metadata?: grpc.Metadata,
+      upStreamRetryCodes?: number[],
     }
   `;
     const chunks = [];
@@ -177,7 +192,7 @@ function generateGrpcWebImpl(ctx, returnObservable, hasStreamingMethods) {
     export class GrpcWebImpl {
       private host: string;
       private options: ${options};
-      
+
       constructor(host: string, options: ${options}) {
         this.host = host;
         this.options = options;
@@ -191,6 +206,7 @@ function generateGrpcWebImpl(ctx, returnObservable, hasStreamingMethods) {
     }
     if (hasStreamingMethods) {
         chunks.push(createInvokeMethod(ctx));
+        chunks.push(createStreamMethod(ctx));
     }
     chunks.push((0, ts_poet_1.code) `}`);
     return (0, ts_poet_1.joinCode)(chunks, { trim: false });
@@ -258,7 +274,7 @@ function createObservableUnaryMethod(ctx) {
           },
         });
       }).pipe(${take}(1));
-    } 
+    }
   `;
 }
 function createInvokeMethod(ctx) {
@@ -268,8 +284,7 @@ function createInvokeMethod(ctx) {
       _request: any,
       metadata: grpc.Metadata | undefined
     ): ${(0, types_1.observableType)(ctx)}<any> {
-      // Status Response Codes (https://developers.google.com/maps-booking/reference/grpc-api/status_codes)
-      const upStreamCodes = [2, 4, 8, 9, 10, 13, 14, 15]; 
+      const upStreamCodes = this.options.upStreamRetryCodes || [];
       const DEFAULT_TIMEOUT_TIME: number = 3_000;
       const request = { ..._request, ...methodDesc.requestType };
       const maybeCombinedMetadata =
@@ -302,3 +317,50 @@ function createInvokeMethod(ctx) {
     }
   `;
 }
+function createStreamMethod(ctx) {
+    return (0, ts_poet_1.code) `
+  stream<T extends MethodDefinitionish>(
+    methodDesc: T,
+    _request: ${(0, types_1.observableType)(ctx)}<any>,
+    metadata: ${grpc}.Metadata | undefined,
+    rpcOptions: ${grpc}.RpcOptions | undefined
+  ): ${(0, types_1.observableType)(ctx)}<any> {
+    const defaultOptions = {
+      host: this.host,
+      debug: rpcOptions?.debug || this.options.debug,
+      transport: rpcOptions?.transport || this.options.streamingTransport || this.options.transport,
+    }
+
+    let started = false;
+    const client = ${grpc}.client(methodDesc, defaultOptions);
+
+    const subscription = _request.subscribe((_req: any) => {
+      const request = { ..._req, ...methodDesc.requestType };
+      if (!started) {
+        client.start(metadata);
+        started = true;
+      }
+      client.send(request);
+    })
+
+    subscription.add(() => {
+      client.finishSend();
+    })
+
+    return new Observable((observer) => {
+      client.onEnd((code: ${grpc}.Code, message: string, trailers: ${grpc}.Metadata) => {
+        subscription.unsubscribe();
+        if (code === 0) {
+          observer.complete();
+        } else {
+          observer.error(new Error(\`Error \${code} \${message}\`));
+        }
+      })
+      client.onMessage((res: any) => {
+        observer.next(res);
+      })
+      observer.add(() => client.close())
+    }).pipe(share());
+  }
+`;
+}

package/build/generate-services.js

@@ -8,7 +8,6 @@ const sourceInfo_1 = require("./sourceInfo");
 const main_1 = require("./main");
 const hash = (0, ts_poet_1.imp)('hash*object-hash');
 const dataloader = (0, ts_poet_1.imp)('DataLoader*dataloader');
-const Reader = (0, ts_poet_1.imp)('Reader@protobufjs/minimal');
 /**
  * Generates an interface for `serviceDesc`.
  *
@@ -43,7 +42,14 @@ function generateService(ctx, fileDesc, sourceInfo, serviceDesc) {
         // Use metadata as last argument for interface only configuration
         if (options.outputClientImpl === 'grpc-web') {
             // We have to use grpc.Metadata where grpc will come from @improbable-eng
-            params.push((0, ts_poet_1.code) `metadata?: grpc.Metadata`);
+            if (methodDesc.clientStreaming) {
+                params.push((0, ts_poet_1.code) `options?: {
+          metadata?: grpc.Metadata,
+          rpcOptions?: grpc.RpcOptions}`);
+            }
+            else {
+                params.push((0, ts_poet_1.code) `metadata?: grpc.Metadata`);
+            }
         }
         else if (options.addGrpcMetadata) {
             const Metadata = (0, ts_poet_1.imp)('Metadata@@grpc/grpc-js');
@@ -76,7 +82,7 @@ exports.generateService = generateService;
 function generateRegularRpcMethod(ctx, fileDesc, serviceDesc, methodDesc) {
     (0, utils_1.assertInstanceOf)(methodDesc, utils_1.FormattedMethodDescriptor);
     const { options, utils } = ctx;
-    const Reader = (0, ts_poet_1.imp)('Reader@protobufjs/minimal');
+    const Reader = (0, utils_1.impFile)(ctx.options, 'Reader@protobufjs/minimal');
     const rawInputType = (0, types_1.rawRequestType)(ctx, methodDesc);
     const inputType = (0, types_1.requestType)(ctx, methodDesc);
     const outputType = (0, types_1.responseType)(ctx, methodDesc);
@@ -223,6 +229,7 @@ function generateCachingRpcMethod(ctx, fileDesc, serviceDesc, methodDesc) {
     const inputType = (0, types_1.requestType)(ctx, methodDesc);
     const outputType = (0, types_1.responseType)(ctx, methodDesc);
     const uniqueIdentifier = `${(0, utils_1.maybePrefixPackage)(fileDesc, serviceDesc.name)}.${methodDesc.name}`;
+    const Reader = (0, utils_1.impFile)(ctx.options, 'Reader@protobufjs/minimal');
     const lambda = (0, ts_poet_1.code) `
     (requests) => {
       const responses = requests.map(async request => {

package/build/generate-type-registry.js

@@ -2,8 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateTypeRegistry = void 0;
 const ts_poet_1 = require("ts-poet");
-const Writer = (0, ts_poet_1.imp)('Writer@protobufjs/minimal');
-const Reader = (0, ts_poet_1.imp)('Reader@protobufjs/minimal');
+const utils_1 = require("./utils");
 function generateTypeRegistry(ctx) {
     const chunks = [];
     chunks.push(generateMessageType(ctx));
@@ -22,6 +21,8 @@ function generateMessageType(ctx) {
     chunks.push((0, ts_poet_1.code) `export interface MessageType<Message extends UnknownMessage = UnknownMessage> {`);
     chunks.push((0, ts_poet_1.code) `$type: Message['$type'];`);
     if (ctx.options.outputEncodeMethods) {
+        const Writer = (0, utils_1.impFile)(ctx.options, 'Writer@protobufjs/minimal');
+        const Reader = (0, utils_1.impFile)(ctx.options, 'Reader@protobufjs/minimal');
         chunks.push((0, ts_poet_1.code) `encode(message: Message, writer?: ${Writer}): ${Writer};`);
         chunks.push((0, ts_poet_1.code) `decode(input: ${Reader} | Uint8Array, length?: number): Message;`);
     }

package/build/main.js

@@ -108,14 +108,13 @@ function generateFile(ctx, fileDesc) {
           };
         `);
             if (options.outputTypeRegistry) {
-                const messageTypeRegistry = (0, ts_poet_1.imp)('messageTypeRegistry@./typeRegistry');
+                const messageTypeRegistry = (0, utils_1.impFile)(options, 'messageTypeRegistry@./typeRegistry');
                 chunks.push((0, ts_poet_1.code) `
             ${messageTypeRegistry}.set(${fullName}.$type, ${fullName});
           `);
             }
         }, options);
     }
-    let hasServerStreamingMethods = false;
     let hasStreamingMethods = false;
     (0, visit_1.visitServices)(fileDesc, sourceInfo, (serviceDesc, sInfo) => {
         if (options.nestJs) {
@@ -155,12 +154,7 @@ function generateFile(ctx, fileDesc) {
                         chunks.push((0, generate_grpc_web_1.generateGrpcClientImpl)(ctx, fileDesc, serviceDesc));
                         chunks.push((0, generate_grpc_web_1.generateGrpcServiceDesc)(fileDesc, serviceDesc));
                         serviceDesc.method.forEach((method) => {
-                            if (!method.clientStreaming) {
-                                chunks.push((0, generate_grpc_web_1.generateGrpcMethodDesc)(ctx, serviceDesc, method));
-                            }
-                            if (method.serverStreaming) {
-                                hasServerStreamingMethods = true;
-                            }
+                            chunks.push((0, generate_grpc_web_1.generateGrpcMethodDesc)(ctx, serviceDesc, method));
                         });
                     }
                 }
@@ -179,7 +173,7 @@ function generateFile(ctx, fileDesc) {
             chunks.push((0, generate_services_1.generateRpcType)(ctx, hasStreamingMethods));
         }
         else if (options.outputClientImpl === 'grpc-web') {
-            chunks.push((0, generate_grpc_web_1.addGrpcWebMisc)(ctx, hasServerStreamingMethods));
+            chunks.push((0, generate_grpc_web_1.addGrpcWebMisc)(ctx, hasStreamingMethods));
         }
     }
     if (options.context) {
@@ -227,8 +221,8 @@ exports.makeUtils = makeUtils;
 function makeLongUtils(options, bytes) {
     // Regardless of which `forceLong` config option we're using, we always use
     // the `long` library to either represent or at least sanity-check 64-bit values
-    const util = (0, ts_poet_1.imp)('util@protobufjs/minimal');
-    const configure = (0, ts_poet_1.imp)('configure@protobufjs/minimal');
+    const util = (0, utils_1.impFile)(options, 'util@protobufjs/minimal');
+    const configure = (0, utils_1.impFile)(options, 'configure@protobufjs/minimal');
     // Before esModuleInterop, we had to use 'import * as Long from long` b/c long is
     // an `export =` module and exports only the Long constructor (which is callable).
     // See https://www.typescriptlang.org/docs/handbook/modules.html#export--and-import--require.
@@ -572,6 +566,7 @@ function generateDecode(ctx, fullName, messageDesc) {
     if (options.usePrototypeForDefaults) {
         createBase = (0, ts_poet_1.code) `Object.create(${createBase}) as ${fullName}`;
     }
+    const Reader = (0, utils_1.impFile)(ctx.options, 'Reader@protobufjs/minimal');
     // create the basic function declaration
     chunks.push((0, ts_poet_1.code) `
     decode(
@@ -712,12 +707,11 @@ function generateDecode(ctx, fullName, messageDesc) {
     chunks.push((0, ts_poet_1.code) `}`);
     return (0, ts_poet_1.joinCode)(chunks, { on: '\n' });
 }
-const Writer = (0, ts_poet_1.imp)('Writer@protobufjs/minimal');
-const Reader = (0, ts_poet_1.imp)('Reader@protobufjs/minimal');
 /** Creates a function to encode a message by loop overing the tags. */
 function generateEncode(ctx, fullName, messageDesc) {
     const { options, utils, typeMap } = ctx;
     const chunks = [];
+    const Writer = (0, utils_1.impFile)(ctx.options, 'Writer@protobufjs/minimal');
     // create the basic function declaration
     chunks.push((0, ts_poet_1.code) `
     encode(

package/build/options.js

@@ -165,6 +165,7 @@ function parseParameter(parameter) {
     return options;
 }
 function getTsPoetOpts(_options) {
-    return { forceModuleImport: ['protobufjs/minimal'] };
+    const imports = ['protobufjs/minimal' + _options.importSuffix];
+    return _options.esModuleInterop ? { forceDefaultImport: imports } : { forceModuleImport: imports };
 }
 exports.getTsPoetOpts = getTsPoetOpts;

package/build/schema.js

@@ -68,7 +68,7 @@ function generateSchema(ctx, fileDesc, sourceInfo) {
         }
     });
     const dependencies = fileDesc.dependency.map((dep) => {
-        return (0, ts_poet_1.code) `${(0, ts_poet_1.imp)(`protoMetadata@./${dep.replace('.proto', '')}`)}`;
+        return (0, ts_poet_1.code) `${(0, utils_1.impFile)(options, `protoMetadata@./${dep.replace('.proto', '')}`)}`;
     });
     // Use toObject so that we get enums as numbers (instead of the default toJSON behavior)
     const descriptor = ts_proto_descriptors_1.FileDescriptorProto.fromPartial(fileDesc);

package/build/types.js

@@ -641,7 +641,7 @@ function responseObservable(ctx, methodDesc) {
 exports.responseObservable = responseObservable;
 function responsePromiseOrObservable(ctx, methodDesc) {
     const { options } = ctx;
-    if (options.returnObservable || methodDesc.serverStreaming) {
+    if (options.returnObservable || methodDesc.serverStreaming || methodDesc.clientStreaming) {
         return responseObservable(ctx, methodDesc);
     }
     return responsePromise(ctx, methodDesc);

package/build/utils.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.impProto = exports.getPropertyAccessor = exports.getFieldJsonName = exports.FormattedMethodDescriptor = exports.assertInstanceOf = exports.maybePrefixPackage = exports.prefixDisableLinter = exports.maybeAddComment = exports.upperFirst = exports.lowerFirst = exports.singular = exports.fail = exports.readToBuffer = exports.protoFilesToGenerate = void 0;
+exports.impProto = exports.impFile = exports.getPropertyAccessor = exports.getFieldJsonName = exports.FormattedMethodDescriptor = exports.assertInstanceOf = exports.maybePrefixPackage = exports.prefixDisableLinter = exports.maybeAddComment = exports.upperFirst = exports.lowerFirst = exports.singular = exports.fail = exports.readToBuffer = exports.protoFilesToGenerate = void 0;
 const ts_poet_1 = require("ts-poet");
 const options_1 = require("./options");
 const case_1 = require("./case");
@@ -169,6 +169,10 @@ function getPropertyAccessor(objectName, propertyName, optional = false) {
         : `${objectName}${optional ? '?.' : ''}[${JSON.stringify(propertyName)}]`;
 }
 exports.getPropertyAccessor = getPropertyAccessor;
+function impFile(options, spec) {
+    return (0, ts_poet_1.imp)(`${spec}${options.importSuffix}`);
+}
+exports.impFile = impFile;
 function impProto(options, module, type) {
     const importString = `${type}@./${module}${options.fileSuffix}${options.importSuffix}`;
     if (options.onlyTypes) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ts-proto",
-  "version": "1.117.0",
+  "version": "1.119.0",
   "description": "",
   "main": "build/plugin.js",
   "repository": "github:stephenh/ts-proto",