ts-proto 1.97.1 → 1.100.0

package/CHANGELOG.md

@@ -1,3 +1,31 @@
+# [1.100.0](https://github.com/stephenh/ts-proto/compare/v1.99.0...v1.100.0) (2022-01-09)
+
+
+### Features
+
+* support mapping ObjectId message as mongodb.ObjectId ([#467](https://github.com/stephenh/ts-proto/issues/467)) ([8b23897](https://github.com/stephenh/ts-proto/commit/8b2389715ecfd5d51b1b24f5a9332e4ff9f09a27))
+
+# [1.99.0](https://github.com/stephenh/ts-proto/compare/v1.98.0...v1.99.0) (2022-01-07)
+
+
+### Features
+
+* yarn watch updates (specified) tests when source files change ([#465](https://github.com/stephenh/ts-proto/issues/465)) ([275d0e7](https://github.com/stephenh/ts-proto/commit/275d0e7c61f3acb2b1fd670b1974e64dd49d6ff4))
+
+# [1.98.0](https://github.com/stephenh/ts-proto/compare/v1.97.2...v1.98.0) (2022-01-06)
+
+
+### Features
+
+* watch for changed integration test files ([#464](https://github.com/stephenh/ts-proto/issues/464)) ([988cd7e](https://github.com/stephenh/ts-proto/commit/988cd7eb84bc3b8b72d6b4d59c38aa794c16c638))
+
+## [1.97.2](https://github.com/stephenh/ts-proto/compare/v1.97.1...v1.97.2) (2022-01-06)
+
+
+### Performance Improvements
+
+* fromJSON returns object literal to allow v8 optimizations ([#463](https://github.com/stephenh/ts-proto/issues/463)) ([5fcd05b](https://github.com/stephenh/ts-proto/commit/5fcd05b79e7c02547c4b6db46fae7a7202f97629))
+
 ## [1.97.1](https://github.com/stephenh/ts-proto/compare/v1.97.0...v1.97.1) (2022-01-05)
 
 

package/README.markdown

@@ -13,6 +13,7 @@
   - [Table of contents](#table-of-contents)
 - [Overview](#overview)
 - [QuickStart](#quickstart)
+    - [Buf](#buf)
 - [Goals](#goals)
 - [Example Types](#example-types)
 - [Highlights](#highlights)
@@ -84,6 +85,19 @@ This will generate `*.ts` source files for the given `*.proto` types.
 
 If you want to package these source files into an npm package to distribute to clients, just run `tsc` on them as usual to generate the `.js`/`.d.ts` files, and deploy the output as a regular npm package.
 
+## Buf
+
+If you're using Buf, pass `strategy: all` in your `buf.gen.yaml` file ([docs](https://docs.buf.build/configuration/v1/buf-gen-yaml#strategy)).
+
+```yaml
+version: v1
+plugins:
+  - name: ts
+    out: ../gen/ts
+    strategy: all
+    path: ../node_modules/ts-proto/protoc-gen-ts_proto
+```    
+
 # Goals
 
 - Idiomatic TypeScript/ES6 types
@@ -161,6 +175,10 @@ creating a class and calling the right getters/setters.
 
 - `fromJSON`/`toJSON` use the [proto3 canonical JSON encoding format](https://developers.google.com/protocol-buffers/docs/proto3#json) (e.g. timestamps are ISO strings), unlike [`protobufjs`](https://github.com/protobufjs/protobuf.js/issues/1304). 
 
+- ObjectIds can be mapped as `mongodb.ObjectId`
+
+  (Configurable with the `useObjectId` parameter.)
+
 # Auto-Batching / N+1 Prevention
 
 (Note: this is currently only supported by the Twirp clients.)
@@ -333,6 +351,8 @@ Generated code will be placed in the Gradle build directory.
 
 - With `--ts_proto_opt=useDate=false`, fields of type `google.protobuf.Timestamp` will not be mapped to type `Date` in the generated types. See [Timestamp](#timestamp) for more details.
 
+- With `--ts_proto_opt=useObjectId=true`, fields of a type called ObjectId where the message is constructed to have on field called value that is a string will be mapped to type `mongodb.ObjectId` in the generated types. This will require your project to install the mongodb npm package. See [ObjectId](#objectid) for more details.
+
 - With `--ts_proto_opt=outputSchema=true`, meta typings will be generated that can later be used in other code generators.
 
 - With `--ts_proto_opt=outputTypeRegistry=true`, the type registry will be generated that can be used to resolve message types by fully-qualified name. Also, each message will get extra `$type` field containing fully-qualified name.
@@ -404,11 +424,13 @@ The commands below assume you have **Docker** installed. To use a **local** copy
 **Workflow**
 
 - Modifying the plugin implementation:
+  - The most important logic is found in [src/main.ts](src/main.ts).
   - Run `yarn bin2ts` or `yarn bin2ts:local`.  
     _Since the proto files were not changed, you only need to regenerate the typescript files._
   - Run `yarn test` to verify the typescript files are compatible with the reference implementation, and pass other tests.
 - Updating or adding `.proto` files in the integration directory:
-  - Run `yarn build:test` to regenerate the integration test files.
+  - Run `yarn watch` to automatically regenerate test files when proto files change. 
+    - Or run `yarn build:test` to regenerate all integration test files. 
   - Run `yarn test` to retest.
 
 **Contributing**
@@ -669,7 +691,6 @@ The representation of `google.protobuf.Timestamp` is configurable by the `useDat
 | --------------------------- | ---------------------- | ------------------------------------ | ---------------- |
 | `google.protobuf.Timestamp` | `Date`                 | `{ seconds: number, nanos: number }` | `string`         |
 
-
 # Number Types
 
 Numbers are by default assumed to be plain JavaScript `number`s.

package/build/main.js

@@ -193,6 +193,7 @@ function makeUtils(options) {
     return {
         ...bytes,
         ...makeDeepPartial(options, longs),
+        ...makeObjectIdMethods(options),
         ...makeTimestampMethods(options, longs),
         ...longs,
         ...makeComparisonUtils(),
@@ -325,6 +326,32 @@ function makeDeepPartial(options, longs) {
     `);
     return { Builtin, DeepPartial, Exact };
 }
+function makeObjectIdMethods(options) {
+    const mongodb = ts_poet_1.imp('mongodb*mongodb');
+    const fromProtoObjectId = ts_poet_1.conditionalOutput('fromProtoObjectId', ts_poet_1.code `
+      function fromProtoObjectId(oid: ObjectId): ${mongodb}.ObjectId {
+        return new ${mongodb}.ObjectId(oid.value);
+      }
+    `);
+    const fromJsonObjectId = ts_poet_1.conditionalOutput('fromJsonObjectId', ts_poet_1.code `
+      function fromJsonObjectId(o: any): ${mongodb}.ObjectId {
+        if (o instanceof ${mongodb}.ObjectId) {
+          return o;
+        } else if (typeof o === "string") {
+          return new ${mongodb}.ObjectId(o);
+        } else {
+          return ${fromProtoObjectId}(ObjectId.fromJSON(o));
+        }
+      }
+    `);
+    const toProtoObjectId = ts_poet_1.conditionalOutput('toProtoObjectId', ts_poet_1.code `
+      function toProtoObjectId(oid: ${mongodb}.ObjectId): ObjectId {
+        const value = oid.toString();
+        return { value };
+      }
+    `);
+    return { fromJsonObjectId, fromProtoObjectId, toProtoObjectId };
+}
 function makeTimestampMethods(options, longs) {
     const Timestamp = utils_1.impProto(options, 'google/protobuf/timestamp', 'Timestamp');
     let seconds = 'date.getTime() / 1_000';
@@ -575,6 +602,10 @@ function generateDecode(ctx, fullName, messageDesc) {
             const type = types_1.basicTypeName(ctx, field, { keepValueType: true });
             readSnippet = ts_poet_1.code `${utils.fromTimestamp}(${type}.decode(reader, reader.uint32()))`;
         }
+        else if (types_1.isObjectId(field) && options.useMongoObjectId) {
+            const type = types_1.basicTypeName(ctx, field, { keepValueType: true });
+            readSnippet = ts_poet_1.code `${utils.fromProtoObjectId}(${type}.decode(reader, reader.uint32()))`;
+        }
         else if (types_1.isMessage(field)) {
             const type = types_1.basicTypeName(ctx, field);
             readSnippet = ts_poet_1.code `${type}.decode(reader, reader.uint32())`;
@@ -659,6 +690,11 @@ function generateEncode(ctx, fullName, messageDesc) {
             const tag = ((field.number << 3) | types_1.basicWireType(field.type)) >>> 0;
             writeSnippet = (place) => ts_poet_1.code `writer.uint32(${tag}).${types_1.toReaderCall(field)}(${place})`;
         }
+        else if (types_1.isObjectId(field) && options.useMongoObjectId) {
+            const tag = ((field.number << 3) | 2) >>> 0;
+            const type = types_1.basicTypeName(ctx, field, { keepValueType: true });
+            writeSnippet = (place) => ts_poet_1.code `${type}.encode(${utils.toProtoObjectId}(${place}), writer.uint32(${tag}).fork()).ldelim()`;
+        }
         else if (types_1.isTimestamp(field) && (options.useDate === options_1.DateOption.DATE || options.useDate === options_1.DateOption.STRING)) {
             const tag = ((field.number << 3) | 2) >>> 0;
             const type = types_1.basicTypeName(ctx, field, { keepValueType: true });
@@ -817,8 +853,12 @@ function generateFromJson(ctx, fullName, messageDesc) {
     // create the basic function declaration
     chunks.push(ts_poet_1.code `
     fromJSON(${messageDesc.field.length > 0 ? 'object' : '_'}: any): ${fullName} {
-      const message = createBase${fullName}();
+      return {
   `);
+    if (ctx.options.outputTypeRegistry) {
+        chunks.push(ts_poet_1.code `$type: ${fullName}.$type,`);
+    }
+    const oneofFieldsCases = messageDesc.oneofDecl.map((oneof, oneofIndex) => messageDesc.field.filter(types_1.isWithinOneOf).filter((field) => field.oneofIndex === oneofIndex));
     // add a check for each incoming field
     messageDesc.field.forEach((field) => {
         const fieldName = case_1.maybeSnakeToCamel(field.name, options);
@@ -848,6 +888,9 @@ function generateFromJson(ctx, fullName, messageDesc) {
                     return ts_poet_1.code `${cstr}(${from})`;
                 }
             }
+            else if (types_1.isObjectId(field) && options.useMongoObjectId) {
+                return ts_poet_1.code `${utils.fromJsonObjectId}(${from})`;
+            }
             else if (types_1.isTimestamp(field) && options.useDate === options_1.DateOption.STRING) {
                 return ts_poet_1.code `String(${from})`;
             }
@@ -897,6 +940,9 @@ function generateFromJson(ctx, fullName, messageDesc) {
                             return ts_poet_1.code `${cstr}(${from})`;
                         }
                     }
+                    else if (types_1.isObjectId(valueType) && options.useMongoObjectId) {
+                        return ts_poet_1.code `${utils.fromJsonObjectId}(${from})`;
+                    }
                     else if (types_1.isTimestamp(valueType) && options.useDate === options_1.DateOption.STRING) {
                         return ts_poet_1.code `String(${from})`;
                     }
@@ -931,62 +977,71 @@ function generateFromJson(ctx, fullName, messageDesc) {
                 const fieldType = types_1.toTypeName(ctx, messageDesc, field);
                 const i = maybeCastToNumber(ctx, messageDesc, field, 'key');
                 chunks.push(ts_poet_1.code `
-          message.${fieldName} = Object.entries(object.${jsonName} ?? {}).reduce<${fieldType}>((acc, [key, value]) => {
-            acc[${i}] = ${readSnippet('value')};
-            return acc;
-          }, {});
+          ${fieldName}: ${ctx.utils.isObject}(object.${jsonName})
+            ? Object.entries(object.${jsonName}).reduce<${fieldType}>((acc, [key, value]) => {
+                acc[${i}] = ${readSnippet('value')};
+                return acc;
+              }, {})
+            : {},
         `);
             }
             else {
                 const readValueSnippet = readSnippet('e');
                 if (readValueSnippet.toString() === ts_poet_1.code `e`.toString()) {
-                    chunks.push(ts_poet_1.code `message.${fieldName} = Array.isArray(object?.${jsonName}) ? [...object.${jsonName}] : [];`);
+                    chunks.push(ts_poet_1.code `${fieldName}: Array.isArray(object?.${jsonName}) ? [...object.${jsonName}] : [],`);
                 }
                 else {
                     // Explicit `any` type required to make TS with noImplicitAny happy. `object` is also `any` here.
                     chunks.push(ts_poet_1.code `
-            message.${fieldName} = Array.isArray(object?.${jsonName}) ? object.${jsonName}.map((e: any) => ${readValueSnippet}): [];
+            ${fieldName}: Array.isArray(object?.${jsonName}) ? object.${jsonName}.map((e: any) => ${readValueSnippet}): [],
           `);
                 }
             }
         }
         else if (types_1.isWithinOneOfThatShouldBeUnion(options, field)) {
-            chunks.push(ts_poet_1.code `if (${ctx.utils.isSet}(object.${jsonName})) {`);
-            const oneofName = case_1.maybeSnakeToCamel(messageDesc.oneofDecl[field.oneofIndex].name, options);
-            chunks.push(ts_poet_1.code `
-        message.${oneofName} = { $case: '${fieldName}', ${fieldName}: ${readSnippet(`object.${jsonName}`)} }
-      `);
-            chunks.push(ts_poet_1.code `}`);
+            const cases = oneofFieldsCases[field.oneofIndex];
+            const firstCase = cases[0];
+            const lastCase = cases[cases.length - 1];
+            if (field === firstCase) {
+                const fieldName = case_1.maybeSnakeToCamel(messageDesc.oneofDecl[field.oneofIndex].name, options);
+                chunks.push(ts_poet_1.code `${fieldName}: `);
+            }
+            const ternaryIf = ts_poet_1.code `${ctx.utils.isSet}(object.${jsonName})`;
+            const ternaryThen = ts_poet_1.code `{ $case: '${fieldName}', ${fieldName}: ${readSnippet(`object.${jsonName}`)}`;
+            chunks.push(ts_poet_1.code `${ternaryIf} ? ${ternaryThen}} : `);
+            if (field === lastCase) {
+                chunks.push(ts_poet_1.code `undefined,`);
+            }
         }
         else if (types_1.isAnyValueType(field)) {
-            chunks.push(ts_poet_1.code `message.${fieldName} = ${ctx.utils.isSet}(object?.${jsonName})
+            chunks.push(ts_poet_1.code `${fieldName}: ${ctx.utils.isSet}(object?.${jsonName})
         ? ${readSnippet(`object.${jsonName}`)}
-        : undefined;
+        : undefined,
       `);
         }
         else if (types_1.isStructType(field)) {
-            chunks.push(ts_poet_1.code `message.${fieldName} = ${ctx.utils.isObject}(object.${jsonName})
+            chunks.push(ts_poet_1.code `${fieldName}: ${ctx.utils.isObject}(object.${jsonName})
           ? ${readSnippet(`object.${jsonName}`)}
-          : undefined;`);
+          : undefined,`);
         }
         else if (types_1.isListValueType(field)) {
             chunks.push(ts_poet_1.code `
-        message.${fieldName} = Array.isArray(object.${jsonName})
+        ${fieldName}: Array.isArray(object.${jsonName})
           ? ${readSnippet(`object.${jsonName}`)}
-          : ${'undefined'};
+          : undefined,
       `);
         }
         else {
             const fallback = types_1.isWithinOneOf(field) ? 'undefined' : types_1.defaultValue(ctx, field);
             chunks.push(ts_poet_1.code `
-        message.${fieldName} = ${ctx.utils.isSet}(object.${jsonName})
+        ${fieldName}: ${ctx.utils.isSet}(object.${jsonName})
           ? ${readSnippet(`object.${jsonName}`)}
-          : ${fallback};
+          : ${fallback},
       `);
         }
     });
     // and then wrap up the switch/while/return
-    chunks.push(ts_poet_1.code `return message`);
+    chunks.push(ts_poet_1.code `};`);
     chunks.push(ts_poet_1.code `}`);
     return ts_poet_1.joinCode(chunks, { on: '\n' });
 }
@@ -1009,6 +1064,9 @@ function generateToJson(ctx, fullName, messageDesc) {
                     ? ts_poet_1.code `${from} !== undefined ? ${toJson}(${from}) : undefined`
                     : ts_poet_1.code `${toJson}(${from})`;
             }
+            else if (types_1.isObjectId(field) && options.useMongoObjectId) {
+                return ts_poet_1.code `${from}.toString()`;
+            }
             else if (types_1.isTimestamp(field) && options.useDate === options_1.DateOption.DATE) {
                 return ts_poet_1.code `${from}.toISOString()`;
             }
@@ -1028,6 +1086,9 @@ function generateToJson(ctx, fullName, messageDesc) {
                 else if (types_1.isBytes(valueType)) {
                     return ts_poet_1.code `${utils.base64FromBytes}(${from})`;
                 }
+                else if (types_1.isObjectId(valueType) && options.useMongoObjectId) {
+                    return ts_poet_1.code `${from}.toString()`;
+                }
                 else if (types_1.isTimestamp(valueType) && options.useDate === options_1.DateOption.DATE) {
                     return ts_poet_1.code `${from}.toISOString()`;
                 }
@@ -1139,6 +1200,9 @@ function generateFromPartial(ctx, fullName, messageDesc) {
             if ((types_1.isLong(field) || types_1.isLongValueType(field)) && options.forceLong === options_1.LongOption.LONG) {
                 return ts_poet_1.code `Long.fromValue(${from})`;
             }
+            else if (types_1.isObjectId(field) && options.useMongoObjectId) {
+                return ts_poet_1.code `${from} as mongodb.ObjectId`;
+            }
             else if (types_1.isPrimitive(field) ||
                 (types_1.isTimestamp(field) && (options.useDate === options_1.DateOption.DATE || options.useDate === options_1.DateOption.STRING)) ||
                 types_1.isValueType(ctx, field)) {
@@ -1165,6 +1229,9 @@ function generateFromPartial(ctx, fullName, messageDesc) {
                     else if (types_1.isAnyValueType(valueType)) {
                         return ts_poet_1.code `${from}`;
                     }
+                    else if (types_1.isObjectId(valueType) && options.useMongoObjectId) {
+                        return ts_poet_1.code `${from} as mongodb.ObjectId`;
+                    }
                     else if (types_1.isTimestamp(valueType) &&
                         (options.useDate === options_1.DateOption.DATE || options.useDate === options_1.DateOption.STRING)) {
                         return ts_poet_1.code `${from}`;

package/build/options.js

@@ -38,6 +38,7 @@ function defaultOptions() {
         forceLong: LongOption.NUMBER,
         useOptionals: 'none',
         useDate: DateOption.DATE,
+        useMongoObjectId: false,
         oneof: OneofOption.PROPERTIES,
         esModuleInterop: false,
         fileSuffix: '',

package/build/types.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.detectBatchMethod = exports.responsePromiseOrObservable = exports.responseObservable = exports.responsePromise = exports.responseType = exports.requestType = exports.rawRequestType = exports.detectMapType = exports.toTypeName = exports.getEnumMethod = exports.messageToTypeName = exports.wrapperTypeName = exports.valueTypeName = exports.isEmptyType = exports.isLongValueType = exports.isStructTypeName = exports.isStructType = exports.isListValueTypeName = exports.isListValueType = exports.isBytesValueType = exports.isAnyValueTypeName = exports.isAnyValueType = exports.isValueType = exports.isTimestamp = exports.isMapType = exports.isWholeNumber = exports.isLong = exports.isRepeated = exports.isWithinOneOfThatShouldBeUnion = exports.isWithinOneOf = exports.isEnum = exports.isMessage = exports.isBytes = exports.isPrimitive = exports.isOptionalProperty = exports.isScalar = exports.createTypeMap = exports.notDefaultCheck = exports.defaultValue = exports.packedType = exports.toReaderCall = exports.basicTypeName = exports.basicLongWireType = exports.basicWireType = void 0;
+exports.detectBatchMethod = exports.responsePromiseOrObservable = exports.responseObservable = exports.responsePromise = exports.responseType = exports.requestType = exports.rawRequestType = exports.detectMapType = exports.toTypeName = exports.getEnumMethod = exports.messageToTypeName = exports.wrapperTypeName = exports.valueTypeName = exports.isEmptyType = exports.isLongValueType = exports.isStructTypeName = exports.isStructType = exports.isListValueTypeName = exports.isListValueType = exports.isBytesValueType = exports.isAnyValueTypeName = exports.isAnyValueType = exports.isValueType = exports.isTimestamp = exports.isObjectId = exports.isMapType = exports.isWholeNumber = exports.isLong = exports.isRepeated = exports.isWithinOneOfThatShouldBeUnion = exports.isWithinOneOf = exports.isEnum = exports.isMessage = exports.isBytes = exports.isPrimitive = exports.isOptionalProperty = exports.isScalar = exports.createTypeMap = exports.notDefaultCheck = exports.defaultValue = exports.packedType = exports.toReaderCall = exports.basicTypeName = exports.basicLongWireType = exports.basicWireType = void 0;
 const ts_proto_descriptors_1 = require("ts-proto-descriptors");
 const ts_poet_1 = require("ts-poet");
 const options_1 = require("./options");
@@ -378,6 +378,11 @@ function isMapType(ctx, messageDesc, field) {
     return detectMapType(ctx, messageDesc, field) !== undefined;
 }
 exports.isMapType = isMapType;
+function isObjectId(field) {
+    // need to use endsWith instead of === because objectid could be imported from an external proto file
+    return field.typeName.endsWith('.ObjectId');
+}
+exports.isObjectId = isObjectId;
 function isTimestamp(field) {
     return field.typeName === '.google.protobuf.Timestamp';
 }
@@ -510,6 +515,10 @@ function messageToTypeName(ctx, protoType, typeOptions = {}) {
             return ts_poet_1.code `string`;
         }
     }
+    // need to use endsWith instead of === because objectid could be imported from an external proto file
+    if (!typeOptions.keepValueType && options.useMongoObjectId && protoType.endsWith('.ObjectId')) {
+        return ts_poet_1.code `mongodb.ObjectId`;
+    }
     const [module, type] = toModuleAndType(typeMap, protoType);
     return ts_poet_1.code `${utils_1.impProto(options, module, type)}`;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ts-proto",
-  "version": "1.97.1",
+  "version": "1.100.0",
   "description": "",
   "main": "build/plugin.js",
   "repository": "github:stephenh/ts-proto",
@@ -19,9 +19,10 @@
     "proto2pbjs:local": "integration/pbjs.sh",
     "bin2ts:local": "integration/codegen.sh",
     "test": "yarn jest -c jest.config.js --maxWorkers=2",
-    "prettier": "prettier --write {src,tests}/**/*.ts",
+    "prettier": "prettier --write {src,tests}/**/*.ts integration/*.ts",
     "prettier:check": "prettier --list-different {src,tests}/**/*.ts",
-    "setup:docker": "docker-compose build"
+    "setup:docker": "docker-compose build",
+    "watch": "ts-node integration/watch.ts"
   },
   "files": [
     "build"
@@ -45,7 +46,9 @@
     "@semantic-release/release-notes-generator": "^9.0.2",
     "@types/jest": "^26.0.22",
     "@types/node": "^14.14.37",
+    "chokidar": "^3.5.2",
     "jest": "^26.6.3",
+    "mongodb": "^4.3.0",
     "prettier": "^2.2.1",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.4.0",