avrotize 2.1.2__tar.gz → 2.2.4__tar.gz

{avrotize-2.1.2 → avrotize-2.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: avrotize
-Version: 2.1.2
+Version: 2.2.4
 Summary: Tools to convert from and to Avro Schema from various other schema languages.
 Author-email: Clemens Vasters <clemensv@microsoft.com>
 Requires-Python: >=3.10
@@ -8,37 +8,39 @@ Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Dist: jsonschema>=4.17.3
+Requires-Dist: jsonschema>=4.23.0
 Requires-Dist: lark>=1.1.9
-Requires-Dist: pyarrow>=15.0.0
-Requires-Dist: asn1tools>=0.166.0
-Requires-Dist: jsonpointer>=2.4
+Requires-Dist: pyarrow>=17.0.0
+Requires-Dist: asn1tools>=0.167.0
+Requires-Dist: jsonpointer>=3.0.0
 Requires-Dist: jsonpath-ng>=1.6.1
 Requires-Dist: jsoncomparison>=1.1.0
-Requires-Dist: requests>=2.31.0
-Requires-Dist: azure-kusto-data>=4.4.1
-Requires-Dist: azure-identity>=1.6.0
+Requires-Dist: requests>=2.32.3
+Requires-Dist: azure-kusto-data>=4.5.1
+Requires-Dist: azure-identity>=1.17.1
 Requires-Dist: datapackage>=1.15.4
 Requires-Dist: jinja2>=3.1.4
-Requires-Dist: pyiceberg>=0.6.1
+Requires-Dist: pyiceberg>=0.7.0
 Requires-Dist: pandas>=2.2.2
-Requires-Dist: pytest>=7.2.1 ; extra == "dev"
-Requires-Dist: fastavro>=1.9.4 ; extra == "dev"
-Requires-Dist: xmlschema>=3.0.2 ; extra == "dev"
-Requires-Dist: xmlunittest>=0.5.0 ; extra == "dev"
-Requires-Dist: pylint>=3.1.0 ; extra == "dev"
-Requires-Dist: dataclasses_json>0.6.4 ; extra == "dev"
-Requires-Dist: dataclasses>=0.6 ; extra == "dev"
-Requires-Dist: pydantic>=2.7.1 ; extra == "dev"
-Requires-Dist: avro>=1.11.3 ; extra == "dev"
-Requires-Dist: testcontainers>=4.4.1 ; extra == "dev"
-Requires-Dist: pymysql>=1.1.0 ; extra == "dev"
+Requires-Dist: docker>=7.1.0
+Requires-Dist: requests==2.32.3
+Requires-Dist: pytest>=8.3.2 ; extra == "dev"
+Requires-Dist: fastavro>=1.9.5 ; extra == "dev"
+Requires-Dist: xmlschema>=3.3.2 ; extra == "dev"
+Requires-Dist: xmlunittest>=1.0.1 ; extra == "dev"
+Requires-Dist: pylint>=3.2.6 ; extra == "dev"
+Requires-Dist: dataclasses_json>=0.6.7 ; extra == "dev"
+Requires-Dist: dataclasses>=0.8 ; extra == "dev"
+Requires-Dist: pydantic>=2.8.2 ; extra == "dev"
+Requires-Dist: avro>=1.12.0 ; extra == "dev"
+Requires-Dist: testcontainers>=4.7.2 ; extra == "dev"
+Requires-Dist: pymysql>=1.1.1 ; extra == "dev"
 Requires-Dist: psycopg2>=2.9.9 ; extra == "dev"
 Requires-Dist: pyodbc>=5.1.0 ; extra == "dev"
-Requires-Dist: pymongo>=4.7.2 ; extra == "dev"
-Requires-Dist: oracledb>=2.2.1 ; extra == "dev"
+Requires-Dist: pymongo>=4.8.0 ; extra == "dev"
+Requires-Dist: oracledb>=2.3.0 ; extra == "dev"
 Requires-Dist: cassandra-driver>=3.29.1 ; extra == "dev"
-Requires-Dist: sqlalchemy>=2.0.15 ; extra == "dev"
+Requires-Dist: sqlalchemy>=2.0.32 ; extra == "dev"
 Provides-Extra: dev
 
 # Avrotize
@@ -56,7 +58,7 @@ The tool leans on the Apache Avro-derived [Avrotize Schema](specs/avrotize-schem
 
 ## Installation
 
-You can install Avrotize from PyPI, [having installed Python 3.11 or later](https://www.python.org/downloads/):
+You can install Avrotize from PyPI, [having installed Python 3.10 or later](https://www.python.org/downloads/):
 
 ```bash
 pip install avrotize

{avrotize-2.1.2 → avrotize-2.2.4}/README.md

@@ -13,7 +13,7 @@ The tool leans on the Apache Avro-derived [Avrotize Schema](specs/avrotize-schem
 
 ## Installation
 
-You can install Avrotize from PyPI, [having installed Python 3.11 or later](https://www.python.org/downloads/):
+You can install Avrotize from PyPI, [having installed Python 3.10 or later](https://www.python.org/downloads/):
 
 ```bash
 pip install avrotize

{avrotize-2.1.2 → avrotize-2.2.4}/avrotize/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.1.2'
-__version_tuple__ = version_tuple = (2, 1, 2)
+__version__ = version = '2.2.4'
+__version_tuple__ = version_tuple = (2, 2, 4)

{avrotize-2.1.2 → avrotize-2.2.4}/avrotize/avrotokusto.py

@@ -3,6 +3,7 @@
 import json
 import sys
 from typing import Any, List
+from avrotize.common import build_flat_type_dict, inline_avro_references, strip_first_doc
 from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
 
 
@@ -13,12 +14,12 @@ class AvroToKusto:
         """Initializes a new instance of the AvroToKusto class."""
         pass
 
-    def convert_record_to_kusto(self, schema: dict, emit_cloudevents_columns: bool, emit_cloudevents_dispatch_table: bool) -> List[str]:
+    def convert_record_to_kusto(self, type_dict:dict, recordschema: dict, emit_cloudevents_columns: bool, emit_cloudevents_dispatch_table: bool) -> List[str]:
         """Converts an Avro record schema to a Kusto table schema."""
         # Get the name and fields of the top-level record
-        table_name = schema["name"]
-        fields = schema["fields"]
-
+        table_name = recordschema["name"]
+        fields = recordschema["fields"]
+        
         # Create a StringBuilder to store the kusto statements
         kusto = []
 
@@ -40,9 +41,11 @@ class AvroToKusto:
         kusto.append("")
 
         # Add the doc string as table metadata
-        if "doc" in schema:
+        if "doc" in recordschema:
+            doc_data = recordschema["doc"]
+            doc_data = (doc_data[:997] + "...") if len(doc_data) > 1000 else doc_data
             doc_string = json.dumps(json.dumps({
-                "description": schema["doc"]
+                "description": doc_data
             }))
             kusto.append(
                 f".alter table [{table_name}] docstring {doc_string};")
@@ -52,11 +55,24 @@ class AvroToKusto:
         for field in fields:
             column_name = field["name"]
             if "doc" in field:
+                doc_data = field["doc"]
+                if len(doc_data) > 900:
+                    doc_data = (doc_data[:897] + "...")
                 doc_content = {
-                    "description": field["doc"]
+                    "description": doc_data
                 }
                 if isinstance(field["type"], list) or isinstance(field["type"], dict):
-                    doc_content["type"] = field["type"]
+                    inline_schema = inline_avro_references(field["type"].copy(), type_dict.copy(), '')
+                    if (len(json.dumps(inline_schema)) + len(doc_data)) > 900:
+                        while strip_first_doc(inline_schema):
+                            if (len(json.dumps(inline_schema)) + len(doc_data)) < 900:
+                                break
+                        if (len(json.dumps(inline_schema)) + len(doc_data)) > 900:
+                            doc_content["schema"] = '{ "doc": "Schema too large to inline. Please refer to the Avro schema for more details." }'
+                        else:
+                            doc_content["schema"] = inline_schema
+                    else:
+                        doc_content["schema"] = inline_schema
                 doc = json.dumps(json.dumps(doc_content))
                 doc_string_statement.append(f"   [{column_name}]: {doc}")
         if doc_string_statement and emit_cloudevents_columns:
@@ -120,6 +136,9 @@ class AvroToKusto:
             kusto.append("]\n```\n\n")
 
         if emit_cloudevents_columns:
+            kusto.append(
+                f".drop materialized-view {table_name}Latest ifexists;")
+            kusto.append("")
             kusto.append(
                 f".create materialized-view with (backfill=true) {table_name}Latest on table {table_name} {{")
             kusto.append(
@@ -128,8 +147,8 @@ class AvroToKusto:
             kusto.append("")
 
         if emit_cloudevents_dispatch_table:
-            event_type = schema["namespace"] + "." + \
-                schema["name"] if "namespace" in schema else schema["name"]
+            event_type = recordschema["namespace"] + "." + \
+                recordschema["name"] if "namespace" in recordschema else recordschema["name"]
 
             query = f"_cloudevents_dispatch | where (specversion == '1.0' and type == '{event_type}') | " + \
                     "project"                        
@@ -224,8 +243,9 @@ class AvroToKusto:
                 "  {\"column\": \"data\", \"path\": \"$.data\"}")
             kusto_script.append("]\n```\n\n")
 
+        type_dict =  build_flat_type_dict(schema)
         for record in schema:
-            kusto_script.extend(self.convert_record_to_kusto(
+            kusto_script.extend(self.convert_record_to_kusto(type_dict,
                 record, emit_cloudevents_columns, emit_cloudevents_dispatch_table))
         return "\n".join(kusto_script)
 

{avrotize-2.1.2 → avrotize-2.2.4}/avrotize/avrotopython/dataclass_core.jinja

@@ -12,17 +12,23 @@ import dataclasses
 {%- if dataclasses_json_annotation %}
 import dataclasses_json
 import json
+{%- for field in fields if field.type == "datetime" or field.type == "typing.Optional[datetime.datetime]" %}
+{%- if loop.first %}
+from marshmallow import fields
+{%- endif %}
+{%- endfor %}
 {%- endif %}
 {%- if avro_annotation %}
 import avro.schema
 import avro.io
 {%- endif %}
-{%- for import_type in import_types %}
-{%- if import_type.startswith('.') %}
-from .{{ import_type.split('.')[-1]|lower }} import {{ import_type.split('.')[-1] }}
-{%- else %}
-from {{ import_type | lower }} import {{ import_type.split('.')[-1] }}
-{%- endif -%}
+{%- for import_type in import_types if import_type not in ['datetime.datetime', 'datetime.date', 'datetime.time', 'datetime.timedelta', 'decimal.Decimal'] %}
+from {{ '.'.join(import_type.split('.')[:-1]) | lower }} import {{ import_type.split('.')[-1] }}
+{%- endfor %}
+{%- for import_type in import_types if import_type in ['datetime.datetime', 'datetime.date', 'datetime.time', 'datetime.timedelta'] %}
+{%- if loop.first %}
+import datetime
+{%- endif %}
 {%- endfor %}
 
 {% if dataclasses_json_annotation %}
@@ -38,8 +44,9 @@ class {{ class_name }}:
         {%- endfor -%}
     """
     {% for field in fields %}
-    {{ field.name }}: {{ field.type }}=dataclasses.field(kw_only=True{% if dataclasses_json_annotation %}, metadata=dataclasses_json.config(field_name="{{ field.original_name }}"){%- endif %})
-    {%- endfor %}    
+    {%- set isdate = field.type == "datetime" or field.type == "typing.Optional[datetime.datetime]" %}
+    {{ field.name }}: {{ field.type }}=dataclasses.field(kw_only=True{% if dataclasses_json_annotation %}, metadata=dataclasses_json.config(field_name="{{ field.original_name }}"{%- if isdate -%}, encoder=lambda d: datetime.datetime.isoformat(d) if d else None, decoder=lambda d:datetime.datetime.fromisoformat(d) if d else None, mm_field=fields.DateTime(format='iso'){%- endif -%}){%- endif %})
+    {%- endfor %}
     {% if avro_annotation %}
     AvroType: typing.ClassVar[avro.schema.Schema] = avro.schema.parse(
         "{{ avro_schema_json }}"
@@ -132,7 +139,9 @@ class {{ class_name }}:
 
         {%- if dataclasses_json_annotation %}
         if content_type == 'application/json':
-            result = json.dumps(self.to_serializer_dict()).encode('utf-8')
+            #pylint: disable=no-member
+            result = self.to_json()
+            #pylint: enable=no-member
         {%- endif %}
 
         if result is not None and content_type.endswith('+gzip'):

avrotize-2.2.4/avrotize/avrotopython/enum_core.jinja

@@ -0,0 +1,87 @@
+from enum import Enum
+
+{%- if ordinals | length == 0 %}
+_{{ class_name }}_members = []
+{%- endif %}
+
+class {{ class_name }}(Enum):
+    """
+    {{ docstring }}
+    """
+    {%- for symbol in symbols %}
+    {{ symbol }} = '{{symbol}}'
+    {%- endfor %}
+
+    @classmethod
+    def from_ordinal(cls, ordinal: int|str) -> '{{ class_name }}':
+        """
+        Get enum member by ordinal
+
+        Args:
+            ordinal (int| str): The ordinal of the enum member. This can be an integer or a string representation of an integer.
+
+        Returns:
+            The enum member corresponding to the ordinal.
+        """
+        {%- if ordinals | length == 0 %}
+        # pylint: disable=global-statement
+        global _{{ class_name }}_members
+        # pylint: enable=global-statement
+        {%- endif %}
+
+        if ordinal is None:
+            raise ValueError("ordinal must not be None")
+        if isinstance(ordinal, str) and ordinal.isdigit():
+            ordinal = int(ordinal)
+        {%- if ordinals | length > 0 %}
+        if isinstance(ordinal, int):
+        {%- for sym, ord in ordinals.items() -%}
+            {%- if loop.first %}
+            if ordinal == {{ ord }}:
+            {%- else %}
+            elif ordinal == {{ ord }}:
+            {%- endif %}
+                return {{ class_name }}.{{ sym }}
+        {%- endfor %}
+            raise ValueError("Ordinal not found in enum")
+        else:
+            raise ValueError("Ordinal must be an integer or a string representation of an integer")
+        {%- else %}
+        if not _{{ class_name }}_members:
+            _{{ class_name }}_members = list(cls)
+        if 0 <= int(ordinal) < len(_{{ class_name }}_members):
+            return _{{ class_name }}_members[ordinal]
+        else:
+            raise IndexError("Ordinal out of range for enum")
+        {%- endif %}
+
+    @classmethod
+    def to_ordinal(cls, member: '{{ class_name }}') -> int:
+        """
+        Get enum ordinal
+
+        Args:
+            member ({{ class_name }}): The enum member to get the ordinal of.
+
+        Returns:
+            The ordinal of the enum member.
+        """
+        {%- if ordinals | length == 0 %}
+        # pylint: disable=global-statement
+        global _{{ class_name }}_members
+        # pylint: enable=global-statement
+        
+        if not _{{ class_name }}_members:
+            _{{ class_name }}_members = list(cls)
+        return _{{ class_name }}_members.index(member)
+        {%- else %}
+        {% for sym, ord in ordinals.items() %}
+        if member == {{ class_name }}.{{ sym }}:
+            return {{ ord }}
+        {%- endfor %}
+        raise ValueError("Member not found in enum")
+        {%- endif %}
+
+{%- if ordinals | length == 0 %}
+_{{ class_name }}_members = list({{ class_name }})
+{%- endif %}

{avrotize-2.1.2 → avrotize-2.2.4}/avrotize/avrotopython/pyproject_toml.jinja

@@ -7,7 +7,7 @@ license = "MIT"
 packages = [ { include = "{{ package_name|replace('-','_') }}", from="src" } ]
 
 [tool.poetry.dependencies]
-python = "<4.0,>=3.11"
+python = "<4.0,>=3.10"
 dataclasses-json = "^0.6.7"
 
 [tool.poetry.dev-dependencies]

{avrotize-2.1.2 → avrotize-2.2.4}/avrotize/avrotopython/test_class.jinja

@@ -8,11 +8,11 @@ import unittest
 
 sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../src'.replace('/', os.sep))))
 
-from {{ base_package }}{{ '.' if base_package else ''}}{{ package_name|lower }}.{{class_name|lower}} import {{ class_name }}
+from {{ package_name | lower }} import {{ class_name }}
 
-{%- for import_type in import_types %}
+{%- for import_type in import_types if import_type not in ['decimal.Decimal', 'datetime.datetime', 'datetime.date', 'datetime.time', 'datetime.timedelta'] %}
 {%- set import_type_name = 'Test_'+import_type.split('.')[-1] %}
-{%- set import_package_name = 'test_'+import_type.replace('.','_') | lower %}
+{%- set import_package_name = 'test_'+'_'.join(import_type.split('.')[:-1]) | lower %}
 
 {%- if import_type.startswith('.') %}
 from .{{ import_package_name }} import {{ import_type_name }}
@@ -20,6 +20,12 @@ from .{{ import_package_name }} import {{ import_type_name }}
 from {{ import_package_name }} import {{ import_type_name }}
 {%- endif -%}
 {%- endfor %}
+{%- for import_type in import_types if import_type in ['datetime.datetime', 'datetime.date', 'datetime.time', 'datetime.timedelta'] %}
+{%- if loop.first %}
+import datetime
+{%- endif %}
+{%- endfor %}
+
 
 class {{ test_class_name }}(unittest.TestCase):
     """

{avrotize-2.1.2 → avrotize-2.2.4}/avrotize/avrotopython/test_enum.jinja

@@ -4,7 +4,7 @@ import unittest
 
 sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '../src'.replace('/', os.sep))))
 
-from {{ base_package }}{{ '.' if base_package else ''}}{{ package_name|lower }}.{{class_name|lower}} import {{ class_name }}
+from {{ package_name | lower }} import {{ class_name }}
 
 class {{ test_class_name }}(unittest.TestCase):
 
@@ -21,4 +21,3 @@ class {{ test_class_name }}(unittest.TestCase):
         """
         return "{{ symbols[0] }}"
 
-