avrotize 2.21.1__py3-none-any.whl

avrotize/prototoavro.py

@@ -0,0 +1,382 @@
+"""
+Module to convert Protobuf .proto files to Avro schema.
+"""
+
+import json
+import os
+import re
+from typing import Dict, List, Tuple
+from avrotize.common import pascal
+from avrotize.dependency_resolver import sort_messages_by_dependencies, inline_dependencies_of
+from . import proto2parser
+from . import proto3parser
+
+AvroSchema = Dict[str, 'AvroSchema'] | List['AvroSchema'] | str | None
+
+class ProtoToAvroConverter:
+    """Class to convert Protobuf .proto files to Avro schema."""
+
+    isomorphic_types = ['float', 'double', 'bytes', 'string']
+
+    def __init__(self, proto_root: str = None):
+        """Initialize ProtoToAvroConverter.
+        
+        Args:
+            proto_root (str): Optional root directory for resolving proto imports. 
+                             When provided, imports are resolved relative to this directory.
+        """
+        self.imported_types: Dict[str, str] = {}
+        self.generated_types: Dict[str, str] = {}
+        self.forward_references: Dict[str, str] = {} # table for resolvbing forward references
+        self.proto_root: str = proto_root
+
+    def proto_type_to_avro_primitive(self, proto_type: str)-> Tuple[bool, str]:
+        """
+        Map Protobuf types to Avro primitive types.
+
+        Args:
+            proto_type (str): Protobuf type to convert.
+
+        Returns:
+            str or dict: Corresponding Avro type.
+        """
+        mapping = {
+            'google.protobuf.Empty': 'null',  # Special handling may be required
+            'bool': 'boolean',
+            'int32': 'int',
+            'uint32': 'int',
+            'sint32': 'int',
+            'int64': 'long',
+            'uint64': 'long',
+            'sint64': 'long',
+            'fixed32': 'int',
+            'fixed64': 'long',
+            'sfixed32': 'int',
+            'sfixed64': 'long',
+            'google.protobuf.Timestamp': {
+                "type": "long",
+                "logicalType": "timestamp-micros"
+            }
+        }
+        if proto_type in self.isomorphic_types:
+            return True, proto_type
+        mapped = mapping.get(proto_type, None)
+        if mapped:
+            return True, mapped
+        return False, proto_type
+
+    def build_forward_references_from_message(self, proto_message_type: proto2parser.Message | proto3parser.Message, avro_namespace: str):
+        """
+        Build forward references from a Protobuf message.
+
+        Args:
+            proto_message_type: The message type from the parsed proto file.
+            avro_namespace (str): The namespace for the message.
+        """
+        for _, nested_message in proto_message_type.messages.items():
+            nested_namespace = avro_namespace + '.' + proto_message_type.name + '_types'
+            self.build_forward_references_from_message(nested_message, nested_namespace)
+        for _, enum_type in proto_message_type.enums.items():
+            nested_namespace = avro_namespace + '.' + proto_message_type.name + '_types'
+            self.forward_references[nested_namespace+'.'+enum_type.name] = "enum"
+        self.forward_references[avro_namespace+'.'+proto_message_type.name] = "record"
+
+    def build_forward_references_from_file(self, proto_file:  proto3parser.ProtoFile| proto2parser.ProtoFile, avro_namespace: str):
+        """
+        Build forward references from a Protobuf file.
+
+        Args:
+            proto_file: The parsed proto file.
+            avro_namespace (str): The namespace for the message.
+        """
+        for _, enum_type in proto_file.enums.items():
+             self.forward_references[avro_namespace+'.'+enum_type.name] = "enum"
+        for _, message in proto_file.messages.items():
+            self.build_forward_references_from_message(message, avro_namespace)
+
+    def convert_proto_to_avro_schema(self, proto_file_path: str, avro_namespace: str, message_type: str) -> list:
+        """
+        Convert .proto file to Avro schema.
+
+        Args:
+            proto_file_path (str): Path to the Protobuf .proto file.
+
+        Returns:
+            list: Avro schema as a list of dictionaries.
+        """
+        with open(proto_file_path, 'r', encoding='utf-8') as proto_file:
+            proto_schema = proto_file.read()
+
+        # Determine whether we have proto3 or proto2 and parse the data
+        if re.search(r'syntax\s*=\s*"proto3"', proto_schema):
+            data: proto3parser.ProtoFile = proto3parser.parse(proto_schema)
+        else:
+            data: proto2parser.ProtoFile = proto2parser.parse(proto_schema)
+
+        # Build forward references
+        self.build_forward_references_from_file(data, avro_namespace)
+        # Avro schema header
+        avro_schema = []
+        for import_ in data.imports:
+            # Handle protobuf imports
+            if import_.startswith('google/protobuf/'):
+                script_path = os.path.dirname(os.path.abspath(__file__))
+                avsc_dir = os.path.join(script_path, 'prototypes')
+                # Load the corresponding avsc file from ./prototypes at this script's path into avro_schema
+                avsc = f'{avsc_dir}/{import_.replace("google/protobuf/", "").replace(".proto", ".avsc")}'
+                with open(avsc, 'r', encoding='utf-8') as avsc_file:
+                    types = json.load(avsc_file)
+                    for t in types:
+                        qualified_name = t["namespace"] + "." + t["name"]
+                        self.imported_types[qualified_name] = t
+            else:
+                # Resolve import path: try proto_root first, then fall back to file-relative path
+                import_path = None
+                
+                if self.proto_root:
+                    # Try resolving relative to proto_root
+                    candidate_path = os.path.join(self.proto_root, import_)
+                    if os.path.exists(candidate_path):
+                        import_path = candidate_path
+                
+                if not import_path:
+                    # Fall back to resolving relative to the directory of the current proto file
+                    cwd = os.path.join(os.getcwd(), os.path.dirname(proto_file_path))
+                    candidate_path = os.path.join(cwd, import_)
+                    if os.path.exists(candidate_path):
+                        import_path = candidate_path
+                
+                # Raise an exception if the imported file does not exist
+                if not import_path:
+                    raise FileNotFoundError(f'Import file {import_} does not exist. Searched in proto_root: {self.proto_root}, and relative to: {os.path.dirname(proto_file_path)}')
+                
+                package_name = pascal(import_.replace('.proto', ''))
+                import_namespace = (avro_namespace + '.' + package_name) if avro_namespace else package_name
+                avro_schema.extend(self.convert_proto_to_avro_schema(import_path, import_namespace, message_type))
+
+
+        # Convert enum fields
+        for _, enum_type in data.enums.items():
+            self.handle_enum(enum_type, avro_schema, avro_namespace)
+
+        # Convert message fields
+        for _, m in data.messages.items():
+            self.handle_message(m, avro_schema, avro_namespace)
+
+
+        # Sort the messages in avro_schema by dependencies
+        if message_type:
+            message_schema = next(
+                (message for message in avro_schema if message['type'] == "record" and message['name'] == message_type), None)
+            if not message_schema:
+                raise ValueError(f'Message type {message_type} not found in the Avro schema.')
+            else:
+                inline_dependencies_of(avro_schema, message_schema)
+                return message_schema
+        else:
+            avro_schema = sort_messages_by_dependencies(avro_schema)
+        return avro_schema
+
+    @staticmethod
+    def clean_comment(comment: str):
+        """
+        Clean comments by stripping slashes, newlines, linefeeds, and extra whitespace.
+
+        Args:
+            comment (str): The comment to clean.
+
+        Returns:
+            str: Cleaned comment.
+        """
+        if comment:
+            return comment.replace('//', '').replace('\n', '').lstrip().rstrip()
+        return None
+
+    def handle_enum(self, enum_type: proto2parser.Enum | proto3parser.Enum, avro_schema: AvroSchema, avro_namespace: str) -> AvroSchema:
+        """
+        Convert enum fields to avro schema.
+
+        Args:
+            enum_type: The enum type from the parsed proto file.
+            avro_schema (list): The list to append the converted enum schema.
+            namespace (str): The namespace for the enum.
+        """
+        comment = self.clean_comment(
+            enum_type.comment.content if enum_type.comment and enum_type.comment.content else None)
+
+        # Create avro schema
+        avro_enum: AvroSchema = {
+            'name': enum_type.name,
+            'type': 'enum',
+            'namespace': avro_namespace,
+            'symbols': [],
+            'ordinals': {}
+        }
+
+        if comment:
+            avro_enum['doc'] = comment
+        for value in enum_type.fields:
+            avro_enum['symbols'].append(value.name)
+            avro_enum['ordinals'][value.name] = int(value.number)
+        avro_schema.append(avro_enum)
+        self.generated_types[avro_enum['namespace']+'.'+avro_enum['name']] = "enum"
+        return avro_enum
+
+    def handle_message(self, proto_message_type: proto2parser.Message | proto3parser.Message, avro_schema: AvroSchema, avro_namespace: str)-> AvroSchema:
+        """
+        Convert protobuf messages to avro records.
+
+        Args:
+            m: The message type from the parsed proto file.
+            avro_schema (list): The list to append the converted message schema.
+            namespace (str): The namespace for the message.
+        """
+        dependencies = []
+
+        comment = self.clean_comment(proto_message_type.comment.content if proto_message_type.comment and proto_message_type.comment.content else None)
+        avro_record: AvroSchema = {
+            'type': 'record',
+            'name': proto_message_type.name,
+            'namespace': avro_namespace,
+            'fields': []
+        }
+        if comment:
+            avro_record['doc'] = comment
+        for proto_field in proto_message_type.fields:
+            avro_type = self.get_avro_type_for_field(proto_message_type, avro_namespace, avro_schema, dependencies, proto_field)
+            comment = self.clean_comment(proto_field.comment.content if proto_field.comment and proto_field.comment.content else None)
+
+            avro_field = {
+                'name': proto_field.name,
+                'type': avro_type,
+            }
+
+            if comment:
+                avro_field['doc'] = comment
+
+            avro_record['fields'].append(avro_field)
+
+        for proto_field in proto_message_type.oneofs:
+            avro_oneof: AvroSchema = {
+                'name': proto_field.name,
+                'type': []
+            }
+            comment = self.clean_comment(proto_field.comment.content if proto_field.comment and proto_field.comment.content else None)
+            if comment:
+                avro_oneof['doc'] = comment
+            for oneof_field in proto_field.fields:
+                avro_type = self.get_avro_type_for_field(proto_message_type, avro_namespace, avro_schema, dependencies, oneof_field)
+                comment = self.clean_comment(oneof_field.comment.content if oneof_field.comment and oneof_field.comment.content else None)
+                if comment:
+                    oneof_field['doc'] = comment
+                avro_oneof['type'].append(avro_type)
+            avro_record['fields'].append(avro_oneof)
+
+        if dependencies:
+            avro_record['dependencies'] = dependencies
+        avro_schema.append(avro_record)
+        for _, nested_message in proto_message_type.messages.items():
+            nested_namespace = avro_namespace + '.' + proto_message_type.name + '_types'
+            self.handle_message(nested_message, avro_schema, nested_namespace)
+        # Convert enum fields
+        for _, enum_type in proto_message_type.enums.items():
+            nested_namespace = avro_namespace + '.' + proto_message_type.name + '_types'
+            self.handle_enum(enum_type, avro_schema, nested_namespace)
+        self.generated_types[avro_record['namespace']+'.'+avro_record['name']] = "record"
+        return avro_record
+
+    def get_avro_type_for_field(self, proto_message_type: proto2parser.Message | proto3parser.Message, avro_namespace: str, avro_schema: AvroSchema, dependencies: List[str], proto_field: proto2parser.Field | proto3parser.Field):
+        """
+        Get Avro type for a Protobuf field.
+
+        Args:
+            m: The message type from the parsed proto file.
+            namespace (str): The namespace for the message.
+            dependencies (list): The list to append the dependencies.
+            f: The field from the parsed proto file.
+
+        Returns:
+            str or dict: Corresponding Avro type.
+        """
+        avro_field_type: AvroSchema = None
+        proto_field_type = proto_field.val_type if proto_field.label == 'repeated' or proto_field.type == 'map' else proto_field.type
+        is_primitive, avro_field_type = self.proto_type_to_avro_primitive(proto_field_type)
+
+        if not is_primitive:
+            if proto_field.type in self.imported_types:
+                avro_field_type = self.imported_types[proto_field.type]
+            else:
+                avro_field_type = avro_namespace + '.' + avro_field_type
+                found_in_nested_definitions = False
+                for k, nested_proto_message_type in proto_message_type.messages.items():
+                    nested_namespace = avro_namespace + '.' + proto_message_type.name + '_types'
+                    if nested_proto_message_type.name == proto_field_type:
+                        avro_field_type = self.handle_message(nested_proto_message_type, avro_schema, nested_namespace)
+                        del proto_message_type.messages[k]
+                        if 'dependencies' in avro_field_type:
+                            dependencies.extend(avro_field_type['dependencies'])
+                            del avro_field_type['dependencies']
+                        found_in_nested_definitions = True
+                        break
+                if not found_in_nested_definitions:
+                    for k, nested_proto_enum_type in proto_message_type.enums.items():
+                        nested_namespace = avro_namespace + '.' + proto_message_type.name + '_types'
+                        if nested_proto_enum_type.name == proto_field_type:
+                            avro_field_type = self.handle_enum(nested_proto_enum_type, avro_schema, nested_namespace)
+                            del proto_message_type.enums[k]
+                            found_in_nested_definitions = True
+                            break
+                if not found_in_nested_definitions:
+                    dependency_avro_field_type = avro_field_type
+                    while '.' in dependency_avro_field_type:
+                        if dependency_avro_field_type in self.forward_references:
+                            dependencies.append(dependency_avro_field_type)
+                            break
+                        n = dependency_avro_field_type.split('.')
+                        dependency_avro_field_type = '.'.join(n[:-2]+[n[-1]])
+
+        if proto_field.label == 'optional':
+            avro_field_type = ["null", avro_field_type]
+        if proto_field.label == 'repeated':
+            avro_type: AvroSchema = {
+                "type": "array",
+                "items": avro_field_type
+            }
+        elif proto_field.type == 'map':
+            avro_type: AvroSchema = {
+                "type": "map",
+                "values": avro_field_type,
+            }
+        else:
+            avro_type = avro_field_type
+        return avro_type
+
+
+def convert_proto_to_avro(proto_file_path: str, avro_schema_path: str, namespace: str = None, message_type: str = None, proto_root: str = None):
+    """
+    Convert Protobuf .proto file to Avro schema.
+
+    Args:
+        proto_file_path (str): Path to the Protobuf .proto file.
+        avro_schema_path (str): Path to save the Avro schema .avsc file.
+        namespace (str): Optional namespace for the Avro schema.
+        message_type (str): Optional specific message type to extract.
+        proto_root (str): Optional root directory for resolving proto imports.
+                         When provided, imports are resolved relative to this directory.
+
+    Raises:
+        FileNotFoundError: If the proto file does not exist.
+        ValueError: If the file extensions are incorrect.
+    """
+    if not os.path.exists(proto_file_path):
+        raise FileNotFoundError(f'Proto file {proto_file_path} does not exist.')
+
+    converter = ProtoToAvroConverter(proto_root=proto_root)
+    if not namespace:
+        namespace = pascal(os.path.basename(proto_file_path).replace('.proto', ''))
+    avro_schema = converter.convert_proto_to_avro_schema(proto_file_path, namespace, message_type)
+
+    # Convert the Avro schema to JSON and write it to the file
+    with open(avro_schema_path, 'w', encoding='utf-8') as avro_file:
+        avro_file.write(json.dumps(avro_schema, indent=2))
+

avrotize/prototypes/any.avsc

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "record",
+    "name": "Any",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "type_url",
+        "type": "string",
+        "doc": "A URL/resource name that uniquely identifies the type of the serialized protocol buffer message. This string must contain at least one \"/\" character. The last segment of the URL's path must represent the fully qualified name of the type (as in `path/google.protobuf.Duration`). The name should be in a canonical form (e.g., leading \".\" is not accepted). In practice, teams usually precompile into the binary all types that they expect it to use in the context of Any. However, for URLs which use the scheme `http`, `https`, or no scheme, one can optionally set up a type server that maps type URLs to message definitions as follows: * If no scheme is provided, `https` is assumed. * An HTTP GET on the URL must yield a [google.protobuf.Type][]   value in binary format, or produce an error. * Applications are allowed to cache lookup results based on the   URL, or have them precompiled into a binary to avoid any   lookup. Therefore, binary compatibility needs to be preserved   on changes to types. (Use versioned type names to manage   breaking changes.) Note: this functionality is not currently available in the official protobuf release, and it is not used for type URLs beginning with type.googleapis.com. As of May 2023, there are no widely used type server implementations and no plans to implement one. Schemes other than `http`, `https` (or the empty scheme) might be used with implementation specific semantics."
+      },
+      {
+        "name": "value",
+        "type": "bytes"
+      }
+    ],
+    "doc": "if (any.is(Foo.class)) {       foo = any.unpack(Foo.class);     }      or ...     if (any.isSameTypeAs(Foo.getDefaultInstance())) {       foo = any.unpack(Foo.getDefaultInstance());     }  Example 3: Pack and unpack a message in Python.     foo = Foo(...)     any = Any()     any.Pack(foo)     ...     if any.Is(Foo.DESCRIPTOR):       any.Unpack(foo)       ...  Example 4: Pack and unpack a message in Go      foo := &pb.Foo{...}      any, err := anypb.New(foo)      if err != nil {        ...      }      ...      foo := &pb.Foo{}      if err := any.UnmarshalTo(foo); err != nil {        ...      } The pack methods provided by protobuf library will by default use 'type.googleapis.com/full.type.name' as the type URL and the unpack methods only use the fully qualified type name after the last '/' in the type URL, for example \"foo.bar.com/x/y.z\" will yield type name \"y.z\". JSON ==== The JSON representation of an `Any` value uses the regular representation of the deserialized, embedded message, with an additional field `@type` which contains the type URL. Example:     package google.profile;     message Person {       string first_name = 1;       string last_name = 2;     }     {       \"@type\": \"type.googleapis.com/google.profile.Person\",       \"firstName\": <string>,       \"lastName\": <string>     } If the embedded message type is well-known and has a custom JSON representation, that representation will be embedded adding a field `value` which holds the custom JSON in addition to the `@type` field. Example (for message [google.protobuf.Duration][]):     {       \"@type\": \"type.googleapis.com/google.protobuf.Duration\",       \"value\": \"1.212s\"     }"
+  }
+]

avrotize/prototypes/api.avsc

@@ -0,0 +1,106 @@
+[
+  {
+    "type": "record",
+    "name": "Method",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "name",
+        "type": "string",
+        "doc": "The simple name of this method."
+      },
+      {
+        "name": "request_type_url",
+        "type": "string",
+        "doc": "A URL of the input message type."
+      },
+      {
+        "name": "request_streaming",
+        "type": "boolean",
+        "doc": "If true, the request is streamed."
+      },
+      {
+        "name": "response_type_url",
+        "type": "string",
+        "doc": "The URL of the output message type."
+      },
+      {
+        "name": "response_streaming",
+        "type": "boolean",
+        "doc": "If true, the response is streamed."
+      },
+      {
+        "name": "options",
+        "type": "repeated",
+        "doc": "Any metadata attached to the method."
+      },
+      {
+        "name": "syntax",
+        "type": "Syntax",
+        "doc": "The source syntax of this method."
+      }
+    ],
+    "doc": "Method represents a method of an API interface."
+  },
+  {
+    "type": "record",
+    "name": "Mixin",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "name",
+        "type": "string",
+        "doc": "The fully qualified name of the interface which is included."
+      },
+      {
+        "name": "root",
+        "type": "string",
+        "doc": "If non-empty specifies a path under which inherited HTTP paths are rooted."
+      }
+    ],
+    "doc": "Declares an API Interface to be included in this interface. The including interface must redeclare all the methods from the included interface, but documentation and options are inherited as follows: - If after comment and whitespace stripping, the documentation   string of the redeclared method is empty, it will be inherited   from the original method. - Each annotation belonging to the service config (http,   visibility) which is not set in the redeclared method will be   inherited. - If an http annotation is inherited, the path pattern will be   modified as follows. Any version prefix will be replaced by the   version of the including interface plus the [root][] path if   specified. Example of a simple mixin:     package google.acl.v1;     service AccessControl {        Get the underlying ACL object.       rpc GetAcl(GetAclRequest) returns (Acl) {         option (google.api.http).get = \"/v1/{resource=**}:getAcl\";       }     }     package google.storage.v2;     service Storage {       rpc GetAcl(GetAclRequest) returns (Acl);        Get a data record.       rpc GetData(GetDataRequest) returns (Data) {         option (google.api.http).get = \"/v2/{resource=**}\";       }     } Example of a mixin configuration:     apis:     - name: google.storage.v2.Storage       mixins:       - name: google.acl.v1.AccessControl The mixin construct implies that all methods in `AccessControl` are also declared with same name and request/response types in `Storage`. A documentation generator or annotation processor will see the effective `Storage.GetAcl` method after inherting documentation and annotations as follows:     service Storage {        Get the underlying ACL object.       rpc GetAcl(GetAclRequest) returns (Acl) {         option (google.api.http).get = \"/v2/{resource=**}:getAcl\";       }       ...     } Note how the version in the path pattern changed from `v1` to `v2`. If the `root` field in the mixin is specified, it should be a relative path under which inherited HTTP paths are placed. Example:     apis:     - name: google.storage.v2.Storage       mixins:       - name: google.acl.v1.AccessControl         root: acls This implies the following inherited HTTP annotation:     service Storage {        Get the underlying ACL object.       rpc GetAcl(GetAclRequest) returns (Acl) {         option (google.api.http).get = \"/v2/acls/{resource=**}:getAcl\";       }       ...     }"
+  },
+  {
+    "type": "record",
+    "name": "Api",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "name",
+        "type": "string",
+        "doc": "The fully qualified name of this interface, including package name followed by the interface's simple name."
+      },
+      {
+        "name": "methods",
+        "type": "repeated",
+        "doc": "The methods of this interface, in unspecified order."
+      },
+      {
+        "name": "options",
+        "type": "repeated",
+        "doc": "Any metadata attached to the interface."
+      },
+      {
+        "name": "version",
+        "type": "string",
+        "doc": "A version string for this interface. If specified, must have the form `major-version.minor-version`, as in `1.10`. If the minor version is omitted, it defaults to zero. If the entire version field is empty, the major version is derived from the package name, as outlined below. If the field is not empty, the version in the package name will be verified to be consistent with what is provided here. The versioning schema uses [semantic versioning](http:semver.org) where the major version number indicates a breaking change and the minor version an additive, non-breaking change. Both version numbers are signals to users what to expect from different versions, and should be carefully chosen based on the product plan. The major version is also reflected in the package name of the interface, which must end in `v<major-version>`, as in `google.feature.v1`. For major versions 0 and 1, the suffix can be omitted. Zero major versions must only be used for experimental, non-GA interfaces."
+      },
+      {
+        "name": "source_context",
+        "type": "SourceContext",
+        "doc": "Source context for the protocol buffer service represented by this message."
+      },
+      {
+        "name": "mixins",
+        "type": "repeated",
+        "doc": "Included interfaces. See [Mixin][]."
+      },
+      {
+        "name": "syntax",
+        "type": "Syntax",
+        "doc": "The source syntax of the service."
+      }
+    ],
+    "doc": "Api is a light-weight descriptor for an API Interface. Interfaces are also described as \"protocol buffer services\" in some contexts, such as by the \"service\" keyword in a .proto file, but they are different from API Services, which represent a concrete implementation of an interface as opposed to simply a description of methods and bindings. They are also sometimes simply referred to as \"APIs\" in other contexts, such as the name of this message itself. See https:cloud.google.com/apis/design/glossary for detailed terminology."
+  }
+]

avrotize/prototypes/duration.avsc

@@ -0,0 +1,20 @@
+[
+  {
+    "type": "record",
+    "name": "Duration",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "seconds",
+        "type": "long",
+        "doc": "Signed seconds of the span of time. Must be from -315,576,000,000 to +315,576,000,000 inclusive. Note: these bounds are computed from: 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years"
+      },
+      {
+        "name": "nanos",
+        "type": "int",
+        "doc": "of time. Durations less than one second are represented with a 0 `seconds` field and a positive or negative `nanos` field. For durations of one second or more, a non-zero value for the `nanos` field must be of the same sign as the `seconds` field. Must be from -999,999,999 to +999,999,999 inclusive."
+      }
+    ],
+    "doc": "end.seconds -= 1;       end.nanos += 1000000000;     } else if (end.nanos >= 1000000000) {       end.seconds += 1;       end.nanos -= 1000000000;     } Example 3: Compute Duration from datetime.timedelta in Python.     td = datetime.timedelta(days=3, minutes=10)     duration = Duration()     duration.FromTimedelta(td) # JSON Mapping In JSON format, the Duration type is encoded as a string rather than an object, where the string ends in the suffix \"s\" (indicating seconds) and is preceded by the number of seconds, with nanoseconds expressed as fractional seconds. For example, 3 seconds with 0 nanoseconds should be encoded in JSON format as \"3s\", while 3 seconds and 1 nanosecond should be expressed in JSON format as \"3.000000001s\", and 3 seconds and 1 microsecond should be expressed in JSON format as \"3.000001s\"."
+  }
+]

avrotize/prototypes/field_mask.avsc

@@ -0,0 +1,18 @@
+[
+  {
+    "type": "record",
+    "name": "FieldMask",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "paths",
+        "type": {
+          "type": "array",
+          "items": "string"
+        },
+        "doc": "The set of field mask paths."
+      }
+    ],
+    "doc": "be appended to the existing repeated field in the target resource. Note that a repeated field is only allowed in the last position of a `paths` string. If a sub-message is specified in the last position of the field mask for an update operation, then new value will be merged into the existing sub-message in the target resource. For example, given the target message:     f {       b {         d: 1         x: 2       }       c: [1]     } And an update message:     f {       b {         d: 10       }       c: [2]     } then if the field mask is:  paths: [\"f.b\", \"f.c\"] then the result will be:     f {       b {         d: 10         x: 2       }       c: [1, 2]     } An implementation may provide options to override this default behavior for repeated and message fields. In order to reset a field's value to the default, the field must be in the mask and set to the default value in the provided resource. Hence, in order to reset all fields of a resource, provide a default instance of the resource and set all fields in the mask, or do not provide a mask as described below. If a field mask is not present on update, the operation applies to all fields (as if a field mask of all fields has been specified). Note that in the presence of schema evolution, this may mean that fields the client does not know and has therefore not filled into the request will be reset to their default. If this is unwanted behavior, a specific service may require a client to always specify a field mask, producing an error if not. As with get operations, the location of the resource which describes the updated values in the request message depends on the operation kind. In any case, the effect of the field mask is required to be honored by the API. ## Considerations for HTTP REST The HTTP kind of an update operation which uses a field mask must be set to PATCH instead of PUT in order to satisfy HTTP semantics (PUT must only be used for full updates). # JSON Encoding of Field Masks In JSON, a field mask is encoded as a single string where paths are separated by a comma. Fields name in each path are converted to/from lower-camel naming conventions. As an example, consider the following message declarations:     message Profile {       User user = 1;       Photo photo = 2;     }     message User {       string display_name = 1;       string address = 2;     } In proto a field mask for `Profile` may look as such:     mask {       paths: \"user.display_name\"       paths: \"photo\"     } In JSON, the same mask is represented as below:     {       mask: \"user.displayName,photo\"     } # Field Masks and Oneof Fields Field masks treat fields in oneofs just as regular fields. Consider the following message:     message SampleMessage {       oneof test_oneof {         string name = 4;         SubMessage sub_message = 9;       }     } The field mask can be:     mask {       paths: \"name\"     } Or:     mask {       paths: \"sub_message\"     } Note that oneof type names (\"test_oneof\" in this case) cannot be used in paths. ## Field Mask Verification The implementation of any API method which has a FieldMask type field in the request should verify the included field paths, and return an `INVALID_ARGUMENT` error if any path is unmappable."
+  }
+]

avrotize/prototypes/struct.avsc

@@ -0,0 +1,60 @@
+[
+  {
+    "type": "record",
+    "name": "Struct",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "fields",
+        "type": {
+          "type": "map",
+          "values": {
+            "type": "record",
+            "name": "Value",
+            "namespace": "google.protobuf",
+            "fields": [
+              {
+                "name": "kind",
+                "type": [
+                  {
+                    "name": "NullValue",
+                    "type": "enum",
+                    "namespace": "google.protobuf",
+                    "symbols": [
+                      "NULL_VALUE"
+                    ],
+                    "doc": "`NullValue` is a singleton enumeration to represent the null value for the `Value` type union. The JSON representation for `NullValue` is JSON `null`."
+                  },
+                  "double",
+                  "string",
+                  "boolean",
+                  "Struct",
+                  {
+                    "type": "record",
+                    "name": "ListValue",
+                    "namespace": "google.protobuf",
+                    "fields": [
+                      {
+                        "name": "values",
+                        "type": {
+                          "type": "array",
+                          "items": "Value"
+                        },
+                        "doc": "Repeated field of dynamically typed values."
+                      }
+                    ],
+                    "doc": "`ListValue` is a wrapper around a repeated field of values. The JSON representation for `ListValue` is JSON array."
+                  }
+                ],
+                "doc": "The kind of value."
+              }
+            ],
+            "doc": "`Value` represents a dynamically typed value which can be either null, a number, a string, a boolean, a recursive struct value, or a list of values. A producer of value is expected to set one of these variants. Absence of any variant indicates an error. The JSON representation for `Value` is JSON value."
+          }
+        },
+        "doc": "Unordered map of dynamically typed values."
+      }
+    ],
+    "doc": "scripting languages like JS a struct is represented as an object. The details of that representation are described together with the proto support for the language. The JSON representation for `Struct` is JSON object."
+  }
+]

avrotize/prototypes/timestamp.avsc

@@ -0,0 +1,20 @@
+[
+  {
+    "type": "record",
+    "name": "Timestamp",
+    "namespace": "google.protobuf",
+    "fields": [
+      {
+        "name": "seconds",
+        "type": "long",
+        "doc": "Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive."
+      },
+      {
+        "name": "nanos",
+        "type": "int",
+        "doc": "second values with fractions must still have non-negative nanos values that count forward in time. Must be from 0 to 999,999,999 inclusive."
+      }
+    ],
+    "doc": "the Joda Time's [`ISODateTimeFormat.dateTime()`]( http:joda-time.sourceforge.net/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime() ) to obtain a formatter capable of generating timestamps in this format."
+  }
+]