pydantic-avro 0.0.3__tar.gz → 0.9.2__tar.gz

pydantic_avro-0.9.2/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.1
+Name: pydantic-avro
+Version: 0.9.2
+Summary: Converting pydantic classes to avro schemas
+Home-page: https://github.com/godatadriven/pydantic-avro
+License: MIT
+Keywords: pydantic,avro
+Author: Peter van 't Hof'
+Author-email: peter.vanthof@godatadriven.com
+Requires-Python: >=3.8.1,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: pydantic (>=1.4,<3.0)
+Project-URL: Repository, https://github.com/godatadriven/pydantic-avro
+Description-Content-Type: text/markdown
+
+[![Python package](https://github.com/godatadriven/pydantic-avro/actions/workflows/python-package.yml/badge.svg)](https://github.com/godatadriven/pydantic-avro/actions/workflows/python-package.yml)
+[![codecov](https://codecov.io/gh/godatadriven/pydantic-avro/branch/main/graph/badge.svg?token=5L08GOERAW)](https://codecov.io/gh/godatadriven/pydantic-avro)
+[![PyPI version](https://badge.fury.io/py/pydantic-avro.svg)](https://badge.fury.io/py/pydantic-avro)
+[![CodeQL](https://github.com/godatadriven/pydantic-avro/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/godatadriven/pydantic-avro/actions/workflows/codeql-analysis.yml)
+
+# pydantic-avro
+
+This library can convert a pydantic class to a avro schema or generate python code from a avro schema.
+
+### Install
+
+```bash
+pip install pydantic-avro
+```
+
+### Pydantic class to avro schema
+
+```python
+import json
+from typing import Optional
+
+from pydantic_avro.base import AvroBase
+
+
+class TestModel(AvroBase):
+    key1: str
+    key2: int
+    key2: Optional[str]
+
+
+schema_dict: dict = TestModel.avro_schema()
+print(json.dumps(schema_dict))
+
+```
+
+### Avro schema to pydantic
+
+```shell
+# Print to stdout
+pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc
+
+# Save it to a file
+pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc --output /path/to/output.py
+```
+
+### Specify expected Avro type
+
+```python
+from datetime import datetime
+from pydantic import Field
+from pydantic_avro.base import AvroBase 
+
+class ExampleModel(AvroBase):
+    field1: int = Field(..., avro_type="long")  # Explicitly set Avro type to "long"
+    field2: datetime = Field(..., avro_type="timestamp-millis")  # Explicitly set Avro type to "timestamp-millis"
+```
+
+### Install for developers
+
+###### Install package
+
+- Requirement: Poetry 1.*
+
+```shell
+poetry install
+```
+
+###### Run unit tests
+```shell
+pytest
+coverage run -m pytest  # with coverage
+# or (depends on your local env) 
+poetry run pytest
+poetry run coverage run -m pytest  # with coverage
+```
+
+##### Run linting
+
+The linting is checked in the github workflow. To fix and review issues run this:
+```shell
+black .   # Auto fix all issues
+isort .   # Auto fix all issues
+pflake .  # Only display issues, fixing is manual
+```
+

pydantic_avro-0.9.2/README.md

@@ -0,0 +1,84 @@
+[![Python package](https://github.com/godatadriven/pydantic-avro/actions/workflows/python-package.yml/badge.svg)](https://github.com/godatadriven/pydantic-avro/actions/workflows/python-package.yml)
+[![codecov](https://codecov.io/gh/godatadriven/pydantic-avro/branch/main/graph/badge.svg?token=5L08GOERAW)](https://codecov.io/gh/godatadriven/pydantic-avro)
+[![PyPI version](https://badge.fury.io/py/pydantic-avro.svg)](https://badge.fury.io/py/pydantic-avro)
+[![CodeQL](https://github.com/godatadriven/pydantic-avro/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/godatadriven/pydantic-avro/actions/workflows/codeql-analysis.yml)
+
+# pydantic-avro
+
+This library can convert a pydantic class to a avro schema or generate python code from a avro schema.
+
+### Install
+
+```bash
+pip install pydantic-avro
+```
+
+### Pydantic class to avro schema
+
+```python
+import json
+from typing import Optional
+
+from pydantic_avro.base import AvroBase
+
+
+class TestModel(AvroBase):
+    key1: str
+    key2: int
+    key2: Optional[str]
+
+
+schema_dict: dict = TestModel.avro_schema()
+print(json.dumps(schema_dict))
+
+```
+
+### Avro schema to pydantic
+
+```shell
+# Print to stdout
+pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc
+
+# Save it to a file
+pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc --output /path/to/output.py
+```
+
+### Specify expected Avro type
+
+```python
+from datetime import datetime
+from pydantic import Field
+from pydantic_avro.base import AvroBase 
+
+class ExampleModel(AvroBase):
+    field1: int = Field(..., avro_type="long")  # Explicitly set Avro type to "long"
+    field2: datetime = Field(..., avro_type="timestamp-millis")  # Explicitly set Avro type to "timestamp-millis"
+```
+
+### Install for developers
+
+###### Install package
+
+- Requirement: Poetry 1.*
+
+```shell
+poetry install
+```
+
+###### Run unit tests
+```shell
+pytest
+coverage run -m pytest  # with coverage
+# or (depends on your local env) 
+poetry run pytest
+poetry run coverage run -m pytest  # with coverage
+```
+
+##### Run linting
+
+The linting is checked in the github workflow. To fix and review issues run this:
+```shell
+black .   # Auto fix all issues
+isort .   # Auto fix all issues
+pflake .  # Only display issues, fixing is manual
+```

{pydantic-avro-0.0.3 → pydantic_avro-0.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pydantic-avro"
-version = "0.0.3"
+version = "0.9.2"
 description = "Converting pydantic classes to avro schemas"
 authors = ["Peter van 't Hof' <peter.vanthof@godatadriven.com>"]
 
@@ -9,28 +9,26 @@ readme = "README.md"
 license = "MIT"
 homepage = "https://github.com/godatadriven/pydantic-avro"
 repository = "https://github.com/godatadriven/pydantic-avro"
-include = [
-    "LICENSE",
-]
 
 
-packages = [{ include = "pydantic_avro", from = "src"}]
+packages = [{ include = "pydantic_avro", from = "src" }]
 
 [tool.poetry.dependencies]
-python = ">=3.6.1,<4.0"
-pydantic = ">=1.0.0"
+python = ">=3.8.1,<4.0"
+pydantic = ">=1.4,<3.0"
 
 [tool.poetry.dev-dependencies]
-coverage= {version=  "^6.1.1", extras=["toml"]}
-pytest= "6.1.0"
-pytest-mock="3.3.1"
-pyproject-flake8 ="^"
-isort ="^5.10.0"
-black="20.8b1"
-pytest-cov= "^2.10.1"
-mypy = "^0.910"
-avro = ""
-fastavro = ""
+coverage = { version = "^7.6.1", extras = ["toml"] }
+pytest = "^8.3.5"
+pytest-mock = "^3.10.0"
+pyproject-flake8 = "^7.0.0"
+isort = "^5.10.0"
+black = "24.8.0"
+pytest-cov = "^5.0.0"
+mypy = "^1.1.1"
+avro = "^1.12.0"
+fastavro = "^1.8.1"
+typing-extensions = "^4.13.2"
 
 [tool.poetry.scripts]
 pydantic-avro = "pydantic_avro.__main__:root_main"
@@ -51,7 +49,7 @@ show_missing = true
 
 [tool.coverage.run]
 command_line = "-m pytest -v tests/"
-omit = ["tests/*",".venv/*"]
+omit = ["tests/*", ".venv/*"]
 
 [tool.flake8]
 max-line-length = 180
@@ -65,6 +63,7 @@ line_length = 120
 [tool.mypy]
 files = "src/"
 python_version = "3.9"
+ignore_missing_imports = "true"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

pydantic_avro-0.9.2/setup.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+package_dir = \
+{'': 'src'}
+
+packages = \
+['pydantic_avro', 'pydantic_avro.from_avro', 'pydantic_avro.to_avro']
+
+package_data = \
+{'': ['*']}
+
+install_requires = \
+['pydantic>=1.4,<3.0']
+
+entry_points = \
+{'console_scripts': ['pydantic-avro = pydantic_avro.__main__:root_main']}
+
+setup_kwargs = {
+    'name': 'pydantic-avro',
+    'version': '0.9.2',
+    'description': 'Converting pydantic classes to avro schemas',
+    'long_description': '[![Python package](https://github.com/godatadriven/pydantic-avro/actions/workflows/python-package.yml/badge.svg)](https://github.com/godatadriven/pydantic-avro/actions/workflows/python-package.yml)\n[![codecov](https://codecov.io/gh/godatadriven/pydantic-avro/branch/main/graph/badge.svg?token=5L08GOERAW)](https://codecov.io/gh/godatadriven/pydantic-avro)\n[![PyPI version](https://badge.fury.io/py/pydantic-avro.svg)](https://badge.fury.io/py/pydantic-avro)\n[![CodeQL](https://github.com/godatadriven/pydantic-avro/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/godatadriven/pydantic-avro/actions/workflows/codeql-analysis.yml)\n\n# pydantic-avro\n\nThis library can convert a pydantic class to a avro schema or generate python code from a avro schema.\n\n### Install\n\n```bash\npip install pydantic-avro\n```\n\n### Pydantic class to avro schema\n\n```python\nimport json\nfrom typing import Optional\n\nfrom pydantic_avro.base import AvroBase\n\n\nclass TestModel(AvroBase):\n    key1: str\n    key2: int\n    key2: Optional[str]\n\n\nschema_dict: dict = TestModel.avro_schema()\nprint(json.dumps(schema_dict))\n\n```\n\n### Avro schema to pydantic\n\n```shell\n# Print to stdout\npydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc\n\n# Save it to a file\npydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc --output /path/to/output.py\n```\n\n### Specify expected Avro type\n\n```python\nfrom datetime import datetime\nfrom pydantic import Field\nfrom pydantic_avro.base import AvroBase \n\nclass ExampleModel(AvroBase):\n    field1: int = Field(..., avro_type="long")  # Explicitly set Avro type to "long"\n    field2: datetime = Field(..., avro_type="timestamp-millis")  # Explicitly set Avro type to "timestamp-millis"\n```\n\n### Install for developers\n\n###### Install package\n\n- Requirement: Poetry 1.*\n\n```shell\npoetry install\n```\n\n###### Run unit tests\n```shell\npytest\ncoverage run -m pytest  # with coverage\n# or (depends on your local env) \npoetry run pytest\npoetry run coverage run -m pytest  # with coverage\n```\n\n##### Run linting\n\nThe linting is checked in the github workflow. To fix and review issues run this:\n```shell\nblack .   # Auto fix all issues\nisort .   # Auto fix all issues\npflake .  # Only display issues, fixing is manual\n```\n',
+    'author': "Peter van 't Hof'",
+    'author_email': 'peter.vanthof@godatadriven.com',
+    'maintainer': 'None',
+    'maintainer_email': 'None',
+    'url': 'https://github.com/godatadriven/pydantic-avro',
+    'package_dir': package_dir,
+    'packages': packages,
+    'package_data': package_data,
+    'install_requires': install_requires,
+    'entry_points': entry_points,
+    'python_requires': '>=3.8.1,<4.0',
+}
+
+
+setup(**setup_kwargs)

pydantic_avro-0.9.2/src/pydantic_avro/__init__.py

@@ -0,0 +1 @@
+from pydantic_avro.to_avro.base import AvroBase

{pydantic-avro-0.0.3 → pydantic_avro-0.9.2}/src/pydantic_avro/__main__.py

@@ -2,7 +2,7 @@ import argparse
 import sys
 from typing import List
 
-from pydantic_avro.avro_to_pydantic import convert_file
+from pydantic_avro.from_avro.avro_to_pydantic import convert_file
 
 
 def main(input_args: List[str]):

pydantic_avro-0.9.2/src/pydantic_avro/base.py

@@ -0,0 +1,2 @@
+# For backwards compatability
+from pydantic_avro.to_avro.base import AvroBase

pydantic_avro-0.9.2/src/pydantic_avro/from_avro/avro_to_pydantic.py

@@ -0,0 +1,48 @@
+import json
+from typing import Optional
+
+from pydantic_avro.from_avro.class_registery import ClassRegistry
+from pydantic_avro.from_avro.types import get_pydantic_type
+
+
+def validate_schema(schema: dict) -> None:
+    if "type" not in schema:
+        raise AttributeError("Type not supported")
+    if "name" not in schema:
+        raise AttributeError("Name is required")
+    if "fields" not in schema:
+        raise AttributeError("Fields are required")
+
+
+def avsc_to_pydantic(schema: dict) -> str:
+    """Generate python code of pydantic of given Avro Schema"""
+    # Ensures that state is clean from previous calls
+    ClassRegistry().clear()
+    validate_schema(schema)
+    get_pydantic_type(schema)
+
+    file_content = """
+from datetime import date, datetime, time
+from decimal import Decimal
+from enum import Enum
+from typing import List, Optional, Dict, Union
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+
+"""
+    file_content += "\n\n".join(ClassRegistry().classes.values())
+
+    return file_content
+
+
+def convert_file(avsc_path: str, output_path: Optional[str] = None):
+    with open(avsc_path, "r") as fh:
+        avsc_dict = json.load(fh)
+    file_content = avsc_to_pydantic(avsc_dict)
+    if output_path is None:
+        print(file_content)
+    else:
+        with open(output_path, "w") as fh:
+            fh.write(file_content)

pydantic_avro-0.9.2/src/pydantic_avro/from_avro/class_registery.py

@@ -0,0 +1,29 @@
+class ClassRegistry:
+    """Singleton class to store generated Pydantic classes."""
+
+    _instance = None
+    _classes: dict = {}
+
+    def __new__(cls):
+        """Singleton implementation."""
+        if cls._instance is None:
+            cls._instance = super(ClassRegistry, cls).__new__(cls)
+            cls._instance._classes = {}
+        return cls._instance
+
+    def add_class(self, name: str, class_def: str):
+        """Add a class to the registry."""
+        self._classes[name] = class_def
+
+    @property
+    def classes(self) -> dict:
+        """Get all classes in the registry."""
+        return self._classes
+
+    def has_class(self, name: str) -> bool:
+        """Check if a class is in the registry."""
+        return name in self._classes.keys()
+
+    def clear(self):
+        """Clear all classes from the registry."""
+        self._classes.clear()

pydantic_avro-0.9.2/src/pydantic_avro/from_avro/types.py

@@ -0,0 +1,164 @@
+import json
+from typing import Callable, Union
+
+from pydantic_avro.from_avro.class_registery import ClassRegistry
+
+LOGICAL_TYPES = {
+    "uuid": "UUID",
+    "decimal": "Decimal",
+    "timestamp-millis": "datetime",
+    "timestamp-micros": "datetime",
+    "time-millis": "time",
+    "time-micros": "time",
+    "date": "date",
+}
+
+
+AVRO_TO_PY_MAPPING = {
+    "string": "str",
+    "int": "int",
+    "long": "int",
+    "boolean": "bool",
+    "double": "float",
+    "float": "float",
+    "bytes": "bytes",
+}
+
+
+def list_type_handler(t: dict) -> str:
+    """Get the Python type of a given Avro list type"""
+    l = t["type"]
+    if "null" in l and len(l) == 2:
+        c = l.copy()
+        c.remove("null")
+        return f"Optional[{get_pydantic_type(c[0])}]"
+    if "null" in l:
+        return f"Optional[Union[{','.join([get_pydantic_type(e) for e in l if e != 'null'])}]]"
+    return f"Union[{','.join([get_pydantic_type(e) for e in l])}]"
+
+
+def map_type_handler(t: dict) -> str:
+    """Get the Python type of a given Avro map type"""
+    type_field = t["type"]
+    value_type = None
+    if isinstance(type_field, dict):
+        avro_value_type = type_field.get("values")
+        if avro_value_type is None:
+            raise AttributeError(f"Values are required for map type. Received: {t}")
+        value_type = get_pydantic_type(avro_value_type)
+    if isinstance(type_field, str):
+        value_type = t.get("values")
+
+    if value_type is None:
+        raise AttributeError(f"Values are required for map type. Received: {t}")
+
+    return f"Dict[str, {value_type}]"
+
+
+def logical_type_handler(t: dict) -> str:
+    """Get the Python type of a given Avro logical type"""
+    return LOGICAL_TYPES[t["type"]["logicalType"]]
+
+
+def enum_type_handler(t: dict) -> str:
+    """Gets the enum type of a given Avro enum type and adds it to the class registry"""
+    if t["type"] == "enum":
+        # comes from a unioned enum (e.g. ["null", "enum"])
+        type_info = t
+    else:
+        # comes from a direct enum
+        type_info = t["type"]
+
+    name = type_info["name"]
+    symbols = type_info["symbols"]
+
+    if not ClassRegistry().has_class(name):
+        enum_class = f"class {name}(str, Enum):\n"
+        for s in symbols:
+            enum_class += f'    {s} = "{s}"\n'
+        ClassRegistry().add_class(name, enum_class)
+    return name
+
+
+def array_type_handler(t: dict) -> str:
+    """Get the Python type of a given Avro array type"""
+    if isinstance(t["type"], dict):
+        sub_type = get_pydantic_type(t["type"]["items"])
+    else:
+        sub_type = get_pydantic_type(t["items"])
+    return f"List[{sub_type}]"
+
+
+def record_type_handler(t: dict) -> str:
+    """Gets the record type of a given Avro record type and adds it to the class registry"""
+    t = t["type"] if isinstance(t["type"], dict) else t
+    name = t["name"]
+    fields = t["fields"] if "fields" in t else t["type"]["fields"]
+    field_strings = [generate_field_string(field) for field in fields]
+    class_body = "\n".join(field_strings) if field_strings else "    pass"
+    current = f"class {name}(BaseModel):\n{class_body}\n"
+    ClassRegistry().add_class(name, current)
+    return name
+
+
+TYPE_HANDLERS = {
+    "list": list_type_handler,
+    "map": map_type_handler,
+    "logical": logical_type_handler,
+    "enum": enum_type_handler,
+    "array": array_type_handler,
+    "record": record_type_handler,
+}
+
+
+def generate_field_string(field: dict) -> str:
+    """Generate a string representing a field in the Pydantic model."""
+    n = field["name"]
+    t = get_pydantic_type(field)
+    default = field.get("default")
+    if field["type"] == "int" and "default" in field and isinstance(default, (bool, type(None))):
+        return f"    {n}: {t} = Field({default}, ge=-2**31, le=(2**31 - 1))"
+    elif field["type"] == "int" and "default" in field:
+        return f"    {n}: {t} = Field({json.dumps(default)}, ge=-2**31, le=(2**31 - 1))"
+    elif field["type"] == "int":
+        return f"    {n}: {t} = Field(..., ge=-2**31, le=(2**31 - 1))"
+    elif "default" not in field:
+        return f"    {n}: {t}"
+    elif isinstance(default, (bool, type(None))):
+        return f"    {n}: {t} = {default}"
+    else:
+        return f"    {n}: {t} = {json.dumps(default)}"
+
+
+def get_pydantic_type(t: Union[str, dict]) -> str:
+    """Get the Pydantic type for a given Avro type"""
+    if isinstance(t, str) or isinstance(t, list):
+        t = {"type": t}
+
+    if isinstance(t.get("type"), str):
+        if ClassRegistry().has_class(t["type"]):
+            return t["type"]
+
+        if t["type"] in AVRO_TO_PY_MAPPING:
+            return AVRO_TO_PY_MAPPING[t["type"]]
+
+    return get_type_handler(t)(t)
+
+
+def get_type_handler(t: dict) -> Callable:
+    """Get the handler for a given Avro type"""
+    h = None
+    t = t["type"]
+    if isinstance(t, str):
+        h = TYPE_HANDLERS.get(t)
+    elif isinstance(t, dict) and "logicalType" in t:
+        h = TYPE_HANDLERS.get("logical")
+    elif isinstance(t, dict) and "type" in t:
+        h = TYPE_HANDLERS.get(t["type"])
+    elif isinstance(t, list):
+        h = TYPE_HANDLERS.get("list")
+
+    if h:
+        return h
+
+    raise NotImplementedError(f"Type {t} not supported yet")

pydantic_avro-0.9.2/src/pydantic_avro/to_avro/base.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from pydantic_avro.to_avro.config import PYDANTIC_V2
+from pydantic_avro.to_avro.types import AvroTypeConverter
+
+
+class AvroBase(BaseModel):
+    """This class provides functionality to convert a pydantic model to an Avro schema."""
+
+    @classmethod
+    def avro_schema(cls, by_alias: bool = True, namespace: Optional[str] = None) -> dict:
+        """Returns the avro schema for the pydantic class
+
+        :param by_alias: generate the schemas using the aliases defined, if any
+        :param namespace: Provide an optional namespace string to use in schema generation
+        :return: dict with the Avro Schema for the model
+        """
+        schema = cls.model_json_schema(by_alias=by_alias) if PYDANTIC_V2 else cls.schema(by_alias=by_alias)
+
+        if namespace is None:
+            # Default namespace will be based on title
+            namespace = schema["title"]
+
+        avro_type_handler = AvroTypeConverter(schema)
+
+        return cls._avro_schema(schema, namespace, avro_type_handler)
+
+    @staticmethod
+    def _avro_schema(schema: dict, namespace: str, avro_type_handler: AvroTypeConverter) -> dict:
+        """Return the avro schema for the given pydantic schema"""
+
+        fields = avro_type_handler.fields_to_avro_dicts(schema)
+        return {"type": "record", "namespace": namespace, "name": schema["title"], "fields": fields}

pydantic_avro-0.9.2/src/pydantic_avro/to_avro/config.py

@@ -0,0 +1,4 @@
+from pydantic import VERSION as PYDANTIC_VERSION
+
+PYDANTIC_V2 = PYDANTIC_VERSION.startswith("2.")
+DEFS_NAME = "$defs" if PYDANTIC_V2 else "definitions"

pydantic_avro-0.9.2/src/pydantic_avro/to_avro/types.py

@@ -0,0 +1,323 @@
+from typing import Any, Dict, List, Optional, Set
+
+from pydantic_avro.to_avro.config import DEFS_NAME
+
+STRING_TYPE_MAPPING = {
+    "date-time": {
+        "type": "long",
+        "logicalType": "timestamp-micros",
+    },
+    "date": {
+        "type": "int",
+        "logicalType": "date",
+    },
+    "time": {
+        "type": "long",
+        "logicalType": "time-micros",
+    },
+    "uuid": {
+        "type": "string",
+        "logicalType": "uuid",
+    },
+    "binary": "bytes",
+}
+
+AVRO_TYPE_MAPPING = {
+    "timestamp-millis": {
+        "type": "long",
+        "logicalType": "timestamp-millis",
+    },
+    "timestamp-micros": {
+        "type": "long",
+        "logicalType": "timestamp-micros",
+    },
+    "time-millis": {
+        "type": "int",
+        "logicalType": "time-millis",
+    },
+    "time-micros": {
+        "type": "long",
+        "logicalType": "time-micros",
+    },
+    "decimal": {
+        "type": "bytes",
+        "logicalType": "decimal",
+    },
+    "uuid": {
+        "type": "string",
+        "logicalType": "uuid",
+    },
+    "int": "int",
+    "long": "long",
+    "float": "float",
+    "double": "double",
+    "boolean": "boolean",
+    "bytes": "bytes",
+    "string": "string",
+    "null": "null",
+    "date": {
+        "type": "int",
+        "logicalType": "date",
+    },
+}
+
+PRIMITVE_TYPES = ["int", "long", "float", "double", "boolean", "null"]
+
+
+def get_definition(ref: str, schema: dict):
+    """Reading definition of base schema for nested structs"""
+    id = ref.replace(f"#/{DEFS_NAME}/", "")
+    d = schema.get(DEFS_NAME, {}).get(id)
+    if d is None:
+        raise RuntimeError(f"Definition {id} does not exist")
+    return d
+
+
+def set_nullability(avro_type_dict: dict) -> dict:
+    """Set the nullability of the field"""
+    if type(avro_type_dict["type"]) is list:
+        if "null" not in avro_type_dict["type"]:
+            avro_type_dict["type"].insert(0, "null")
+    elif avro_type_dict.get("default") is None:
+        avro_type_dict["type"] = ["null", avro_type_dict["type"]]
+    avro_type_dict.setdefault("default", None)
+    return avro_type_dict
+
+
+def null_to_first_element(avro_type_dict: dict) -> dict:
+    """Set the null as the first element in the list as per avro schema requirements"""
+    if type(avro_type_dict["type"]) is list and "null" in avro_type_dict["type"]:
+        avro_type_dict["type"].remove("null")
+        avro_type_dict["type"].insert(0, "null")
+    return avro_type_dict
+
+
+class AvroTypeConverter:
+    """Converts Pydantic schema to AVRO schema."""
+
+    def __init__(self, schema: dict):
+        self.root_schema = schema
+        self.classes_seen: Set[str] = set()
+
+    def fields_to_avro_dicts(self, parent_schema: dict) -> List[dict]:
+        """Converts fields from the schema to AVRO and returns them as a list of dictionaries.
+
+        :param parent_schema: The parent schema of the field (not the root schema for nested models)
+        """
+        fields = []
+
+        required = parent_schema.get("required", [])
+        for name, field_props in parent_schema.get("properties", {}).items():
+            avro_type_dict = self._get_avro_type_dict(field_props=field_props)
+            avro_type_dict["name"] = name
+            if name not in required:
+                set_nullability(avro_type_dict)
+                avro_type_dict = null_to_first_element(avro_type_dict)
+
+            fields.append(avro_type_dict)
+        return fields
+
+    def _get_avro_type_dict(self, field_props: dict) -> dict:
+        """Returns a type of a single field"""
+        avro_type_dict: Dict[str, Any] = {}
+        if "default" in field_props:
+            avro_type_dict["default"] = field_props.get("default")
+        if "description" in field_props:
+            avro_type_dict["doc"] = field_props.get("description")
+        avro_type_dict = self._get_avro_type(field_props, avro_type_dict)
+        return avro_type_dict
+
+    def _get_avro_type(self, field_props: dict, avro_type_dict: dict) -> dict:
+        """Returns AVRO type of a single field"""
+        t = field_props.get("type")
+        f = field_props.get("format")
+        r = field_props.get("$ref")
+        at = field_props.get("avro_type")
+        if "allOf" in field_props and len(field_props["allOf"]) == 1:
+            r = field_props["allOf"][0]["$ref"]
+        if ("prefixItems" in field_props or ("minItems" in field_props and "maxItems" in field_props)) and t == "array":
+            t = "tuple"
+        u = field_props.get("anyOf")
+        o = field_props.get("oneOf")
+        discriminator = field_props.get("discriminator")
+
+        if u is not None:
+            return self._union_to_avro(u, avro_type_dict, discriminator)
+        elif o is not None:
+            return self._union_to_avro(o, avro_type_dict, discriminator)
+        elif r is not None:
+            return self._handle_references(r, avro_type_dict)
+        elif t is None:
+            raise ValueError(f"Field '{field_props}' does not have a defined type.")
+        elif at is not None:
+            if not isinstance(at, str) or at not in AVRO_TYPE_MAPPING:
+                raise ValueError(
+                    f"Field '{field_props}' does not have a supported avro_type. Type should be one of "
+                    f" {AVRO_TYPE_MAPPING.keys()}"
+                )
+            avro_type_dict["type"] = AVRO_TYPE_MAPPING[at]
+        elif t == "array":
+            return self._array_to_avro(field_props, avro_type_dict)
+        elif t == "string":
+            avro_type_dict["type"] = self._string_to_avro(f)
+        elif t == "integer":
+            avro_type_dict["type"] = self._integer_to_avro(field_props)
+        elif t == "object":
+            avro_type_dict["type"] = self._object_to_avro(field_props)
+        elif t == "number":
+            avro_type_dict["type"] = "double"
+        elif t == "boolean":
+            avro_type_dict["type"] = "boolean"
+        elif t == "null":
+            avro_type_dict["type"] = "null"
+        elif t == "tuple":
+            return self._tuple_to_avro(field_props, avro_type_dict)
+        else:
+            raise NotImplementedError(
+                f"Type '{t}' not support yet, "
+                f"please report this at https://github.com/godatadriven/pydantic-avro/issues"
+            )
+
+        return avro_type_dict
+
+    def _handle_references(self, r: str, avro_type_dict: dict) -> dict:
+        """Finds the type of a reference field"""
+        class_name = r.replace(f"#/{DEFS_NAME}/", "")
+        if class_name in self.classes_seen:
+            avro_type_dict["type"] = class_name
+            return avro_type_dict
+
+        d = get_definition(r, self.root_schema)
+        if "enum" in d:
+            avro_type_dict["type"] = {
+                "type": "enum",
+                "symbols": [str(v) for v in d["enum"]],
+                "name": d["title"],
+            }
+        else:
+            avro_type_dict["type"] = {
+                "type": "record",
+                "fields": self.fields_to_avro_dicts(d),
+                # Name of the struct should be unique to the complete schema
+                # Because of this the path in the schema is tracked and used as name for a nested struct/array
+                "name": class_name,
+            }
+
+        self.classes_seen.add(class_name)
+        return avro_type_dict
+
+    @staticmethod
+    def _string_to_avro(f: Optional[str]):
+        """Returns a type of a string field"""
+        if not f:
+            return "string"
+        return STRING_TYPE_MAPPING[f]
+
+    @staticmethod
+    def _integer_to_avro(field_props: dict) -> str:
+        """Returns a type of an integer field"""
+        minimum = field_props.get("minimum")
+        maximum = field_props.get("maximum")
+        # integer in python can be a long, only if minimum and maximum value is set an int can be used
+        if minimum is not None and minimum >= -(2**31) and maximum is not None and maximum <= (2**31 - 1):
+            return "int"
+        return "long"
+
+    def _object_to_avro(self, field_props: dict) -> dict:
+        """Returns a type of an object field"""
+        a = field_props.get("additionalProperties")
+        if not a or isinstance(a, bool):
+            value_type = "string"
+        else:
+            value_type = self._get_avro_type_dict(a)["type"]
+        return {"type": "map", "values": value_type}
+
+    def _union_to_avro(self, field_props: list, avro_type_dict: dict, discriminator: Optional[dict] = None) -> dict:
+        """Returns a type of a union field, including discriminated unions"""
+        # Handle discriminated unions
+        if discriminator is not None:
+            return self._discriminated_union_to_avro(field_props, avro_type_dict, discriminator)
+
+        # Standard union handling (unchanged)
+        avro_type_dict["type"] = []
+        for union_element in field_props:
+            t = self._get_avro_type_dict(union_element)
+            avro_type_dict["type"].append(t["type"])
+        return avro_type_dict
+
+    def _tuple_to_avro(self, field_props: dict, avro_type_dict: dict) -> dict:
+        """Returns a type of a tuple field"""
+        prefix_items = field_props.get("prefixItems")
+        if not prefix_items:
+            # Pydantic v1 there is no prefixItems, but minItems and maxItems and items is a list.
+            prefix_items = field_props.get("items")
+            if not prefix_items:
+                raise ValueError(f"Tuple Field '{field_props}' does not have any items .")
+        possible_types = []
+        for prefix_item in prefix_items:
+            item_type = self._get_avro_type(prefix_item, avro_type_dict).get("type")
+            if not item_type:
+                raise ValueError(f"Field '{avro_type_dict}' does not have a defined type.")
+            if isinstance(item_type, list):
+                possible_types.extend([x for x in item_type if x not in possible_types])
+            elif item_type not in possible_types:
+                possible_types.append(item_type)
+        avro_type_dict["type"] = {"type": "array", "items": possible_types}
+        return avro_type_dict
+
+    def _array_to_avro(self, field_props: dict, avro_type_dict: dict) -> dict:
+        """Returns a type of an array field"""
+        items = field_props.get("items")
+        # In pydantic v1. items is a list, we need to handle this case first
+        # E.g. [{'type': 'number'}, {'type': 'number'}]}
+        if isinstance(items, list):
+            if not len(items):
+                raise ValueError(f"Field '{field_props}' does not have valid items.")
+            items = items[0]
+        if not isinstance(items, dict):
+            raise ValueError(f"Field '{field_props}' does not have valid items.")
+        tn = self._get_avro_type_dict(items)
+        # If items in array are an object:
+        if "$ref" in items:
+            tn = tn["type"]
+        # If items in array are a logicalType
+        if isinstance(tn, dict) and isinstance(tn.get("type", None), (dict, list)):
+            tn = tn["type"]
+        avro_type_dict["type"] = {"type": "array", "items": tn}
+        return avro_type_dict
+
+    def _discriminated_union_to_avro(self, variants: list, avro_type_dict: dict, discriminator: dict) -> dict:
+        """Handles discriminated unions with a discriminator field"""
+        discriminator_property = discriminator.get("propertyName", "type")
+        variant_types = []
+
+        # Process each variant in the union
+        for variant in variants:
+            # Get the discriminator value for this variant
+            disc_value = None
+            if "properties" in variant and discriminator_property in variant.get("properties", {}):
+                prop = variant["properties"][discriminator_property]
+                if "const" in prop:
+                    disc_value = prop["const"]
+                elif "enum" in prop and len(prop["enum"]) == 1:
+                    disc_value = prop["enum"][0]
+
+            # If we can't determine the discriminator value, generate a regular record
+            if disc_value is None:
+                variant_type = self._get_avro_type_dict(variant)
+            else:
+                # Create a named record for this variant
+                variant_name = f"{disc_value}Variant"
+                if "$ref" in variant:
+                    ref_schema = get_definition(variant["$ref"], self.root_schema)
+                    variant_fields = self.fields_to_avro_dicts(ref_schema)
+                else:
+                    variant_fields = self.fields_to_avro_dicts(variant)
+
+                variant_type = {"type": {"type": "record", "name": variant_name, "fields": variant_fields}}
+
+            variant_types.append(variant_type["type"])
+
+        # Set the union of all variant types
+        avro_type_dict["type"] = variant_types
+        return avro_type_dict

pydantic-avro-0.0.3/PKG-INFO

@@ -1,59 +0,0 @@
-Metadata-Version: 2.1
-Name: pydantic-avro
-Version: 0.0.3
-Summary: Converting pydantic classes to avro schemas
-Home-page: https://github.com/godatadriven/pydantic-avro
-License: MIT
-Keywords: pydantic,avro
-Author: Peter van 't Hof'
-Author-email: peter.vanthof@godatadriven.com
-Requires-Python: >=3.6.1,<4.0
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Requires-Dist: pydantic (>=1.0.0)
-Project-URL: Repository, https://github.com/godatadriven/pydantic-avro
-Description-Content-Type: text/markdown
-
-# pydantic-avro
-
-This library can convert a pydantic class to a avro schema or generate python code from a avro schema.
-
-### Install
-
-```bash
-pip install pydantic-avro
-```
-
-### Pydantic class to avro schema
-
-```python
-import json
-from typing import Optional
-
-from pydantic_avro.base import AvroBase
-
-class TestModel(AvroBase):
-    key1: str
-    key2: int
-    key2: Optional[str]
-
-schema_dict: dict = TestModel.avro_schema()
-print(json.dumps(schema_dict))
-
-```
-
-### Avro schema to pydantic
-
-```bash
-#!/usr/bin/env bash
-# Print to stdout
-pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc
-
-# Save it to a file
-pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc --output /path/to/output.py
-```
-

pydantic-avro-0.0.3/README.md

@@ -1,38 +0,0 @@
-# pydantic-avro
-
-This library can convert a pydantic class to a avro schema or generate python code from a avro schema.
-
-### Install
-
-```bash
-pip install pydantic-avro
-```
-
-### Pydantic class to avro schema
-
-```python
-import json
-from typing import Optional
-
-from pydantic_avro.base import AvroBase
-
-class TestModel(AvroBase):
-    key1: str
-    key2: int
-    key2: Optional[str]
-
-schema_dict: dict = TestModel.avro_schema()
-print(json.dumps(schema_dict))
-
-```
-
-### Avro schema to pydantic
-
-```bash
-#!/usr/bin/env bash
-# Print to stdout
-pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc
-
-# Save it to a file
-pydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc --output /path/to/output.py
-```

pydantic-avro-0.0.3/setup.py

@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-package_dir = \
-{'': 'src'}
-
-packages = \
-['pydantic_avro']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['pydantic>=1.0.0']
-
-entry_points = \
-{'console_scripts': ['pydantic-avro = pydantic_avro.__main__:root_main']}
-
-setup_kwargs = {
-    'name': 'pydantic-avro',
-    'version': '0.0.3',
-    'description': 'Converting pydantic classes to avro schemas',
-    'long_description': '# pydantic-avro\n\nThis library can convert a pydantic class to a avro schema or generate python code from a avro schema.\n\n### Install\n\n```bash\npip install pydantic-avro\n```\n\n### Pydantic class to avro schema\n\n```python\nimport json\nfrom typing import Optional\n\nfrom pydantic_avro.base import AvroBase\n\nclass TestModel(AvroBase):\n    key1: str\n    key2: int\n    key2: Optional[str]\n\nschema_dict: dict = TestModel.avro_schema()\nprint(json.dumps(schema_dict))\n\n```\n\n### Avro schema to pydantic\n\n```bash\n#!/usr/bin/env bash\n# Print to stdout\npydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc\n\n# Save it to a file\npydantic-avro avro_to_pydantic --asvc /path/to/schema.asvc --output /path/to/output.py\n```\n',
-    'author': "Peter van 't Hof'",
-    'author_email': 'peter.vanthof@godatadriven.com',
-    'maintainer': None,
-    'maintainer_email': None,
-    'url': 'https://github.com/godatadriven/pydantic-avro',
-    'package_dir': package_dir,
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'entry_points': entry_points,
-    'python_requires': '>=3.6.1,<4.0',
-}
-
-
-setup(**setup_kwargs)

pydantic-avro-0.0.3/src/pydantic_avro/avro_to_pydantic.py

@@ -1,107 +0,0 @@
-import json
-from typing import Optional, Union
-
-
-def avsc_to_pydatic(schema: dict) -> str:
-    """Generate python code of pydantic of given Avro Schema"""
-    if "type" not in schema or schema["type"] != "record":
-        raise AttributeError("Type not supported")
-    if "name" not in schema:
-        raise AttributeError("Name is required")
-    if "fields" not in schema:
-        raise AttributeError("fields are required")
-
-    classes = {}
-
-    def get_python_type(t: Union[str, dict]) -> str:
-        """Returns python type for given avro type"""
-        optional = False
-        if isinstance(t, str):
-            if t == "string":
-                py_type = "str"
-            elif t == "long" or t == "int":
-                py_type = "int"
-            elif t == "boolean":
-                py_type = "bool"
-            elif t == "double" or t == "float":
-                py_type = "float"
-            elif t in classes:
-                py_type = t
-            else:
-                raise NotImplementedError(f"Type {t} not supported yet")
-        elif isinstance(t, list):
-            if "null" in t:
-                optional = True
-            if len(t) > 2 or (not optional and len(t) > 1):
-                raise NotImplementedError("Only a single type ia supported yet")
-            c = t.copy()
-            c.remove("null")
-            py_type = get_python_type(c[0])
-        elif t.get("logicalType") == "uuid":
-            py_type = "UUID"
-        elif t.get("logicalType") == "decimal":
-            py_type = "Decimal"
-        elif t.get("logicalType") == "timestamp-millis" or t.get("logicalType") == "timestamp-micros":
-            py_type = "datetime"
-        elif t.get("logicalType") == "time-millis" or t.get("logicalType") == "time-micros":
-            py_type = "time"
-        elif t.get("logicalType") == "date":
-            py_type = "date"
-        elif t.get("type") == "enum":
-            # TODO: implement Python enum optional
-            py_type = "str"
-        elif t.get("type") == "string":
-            py_type = "str"
-        elif t.get("type") == "array":
-            sub_type = get_python_type(t.get("items"))
-            py_type = f"List[{sub_type}] = []"
-        elif t.get("type") == "record":
-            record_type_to_pydantic(t)
-            py_type = t.get("name")
-        else:
-            raise NotImplementedError(f"Type {t} not supported yet")
-        if optional:
-            return f"Optional[{py_type}] = None"
-        else:
-            return py_type
-
-    def record_type_to_pydantic(schema: dict):
-        """Convert a single avro record type to a pydantic class"""
-        name = schema["name"]
-        current = f"class {name}(BaseModel):\n"
-
-        for field in schema["fields"]:
-            n = field["name"]
-            t = get_python_type(field["type"])
-            current += f"    {n}: {t}\n"
-        if len(schema["fields"]) == 0:
-            current += "    pass\n"
-
-        classes[name] = current
-
-    record_type_to_pydantic(schema)
-
-    file_content = """
-from datetime import date, datetime, time
-from decimal import Decimal
-from typing import List, Optional
-from uuid import UUID
-
-from pydantic import BaseModel
-
-
-"""
-    file_content += "\n\n".join(classes.values())
-
-    return file_content
-
-
-def convert_file(avsc_path: str, output_path: Optional[str] = None):
-    with open(avsc_path, "r") as fh:
-        avsc_dict = json.load(fh)
-    file_content = avsc_to_pydatic(avsc_dict)
-    if output_path is None:
-        print(file_content)
-    else:
-        with open(output_path, "w") as fh:
-            fh.write(file_content)

pydantic-avro-0.0.3/src/pydantic_avro/base.py

@@ -1,105 +0,0 @@
-from typing import List
-
-from pydantic import BaseModel
-
-
-class AvroBase(BaseModel):
-    """This is base pydantic class that will add some methods"""
-
-    @classmethod
-    def avro_schema(cls) -> dict:
-        """Return the avro schema for the pydantic class"""
-        schema = cls.schema()
-        return cls._avro_schema(schema)
-
-    @staticmethod
-    def _avro_schema(schema: dict) -> dict:
-        """Return the avro schema for the given pydantic schema"""
-
-        def get_definition(ref: str, schema: dict, prefix: str):
-            """Reading definition of base schema for nested structs"""
-            id = ref.replace("#/definitions/", "")
-            d = schema.get("definitions", {}).get(id)
-            if d is None:
-                raise RuntimeError(f"Definition {id} does not exist")
-            return get_fields(d, prefix)
-
-        def get_type(key: str, value: dict, prefix: str) -> dict:
-            """Returns a type of a single field"""
-            if len(prefix) > 0:
-                prefix += "_"
-            t = value.get("type")
-            f = value.get("format")
-            r = value.get("$ref")
-            avro_type_dict = {}
-            if r is not None:
-                prefix += key
-                avro_type_dict["type"] = {
-                    "type": "record",
-                    "fields": get_definition(r, schema, prefix),
-                    # Name of the struct should be unique true the complete schema
-                    # Because of this the path in the schema is tracked and used as name for a nested struct/array
-                    "name": prefix + "_" + r.replace("#/definitions/", ""),
-                }
-            elif t == "array":
-                items = value.get("items")
-                tn = get_type(key, items, prefix)
-                # If items in array are a object:
-                if "$ref" in items:
-                    tn = tn["type"]
-                # If items in array are a logicalType
-                if isinstance(tn.get("type", {}), dict) and tn.get("type", {}).get("logicalType") is not None:
-                    tn = tn["type"]
-                avro_type_dict["type"] = {"type": "array", "items": tn}
-            elif t == "string" and f == "date-time":
-                avro_type_dict["type"] = {
-                    "type": "long",
-                    "logicalType": "timestamp-micros",
-                }
-            elif t == "string" and f == "date":
-                avro_type_dict["type"] = {
-                    "type": "int",
-                    "logicalType": "date",
-                }
-            elif t == "string" and f == "time":
-                avro_type_dict["type"] = {
-                    "type": "long",
-                    "logicalType": "time-micros",
-                }
-            elif t == "string" and f == "uuid":
-                avro_type_dict["type"] = {
-                    "type": "string",
-                    "logicalType": "uuid",
-                }
-            elif t == "string":
-                avro_type_dict["type"] = "string"
-            elif t == "number":
-                avro_type_dict["type"] = "double"
-            elif t == "integer":
-                # integer in python can be a long
-                avro_type_dict["type"] = "long"
-            elif t == "boolean":
-                avro_type_dict["type"] = "boolean"
-            else:
-                raise RuntimeError(f"Unknown type found: '{t}'")
-            return avro_type_dict
-
-        def get_fields(s: dict, prefix: str = "") -> List[dict]:
-            """Return a list of fields of a struct"""
-            fields = []
-
-            required = s.get("required", [])
-            for key, value in s.get("properties", {}).items():
-                avro_type_dict = get_type(key, value, prefix)
-                avro_type_dict["name"] = key
-
-                if key not in required:
-                    avro_type_dict["type"] = [avro_type_dict["type"], "null"]
-                    avro_type_dict["default"] = None
-
-                fields.append(avro_type_dict)
-            return fields
-
-        fields = get_fields(schema)
-
-        return {"type": "record", "namespace": schema["title"], "name": schema["title"], "fields": fields}