avrotize 2.21.1__py3-none-any.whl

avrotize/avrotoiceberg.py

@@ -0,0 +1,210 @@
+"""Convert an Avro schema to an Iceberg schema."""
+
+import json
+import sys
+from typing import Dict, List
+import pyarrow as pa
+from pyiceberg.schema import Schema, NestedField
+from pyiceberg.types import (
+    BooleanType,
+    IntegerType,
+    LongType,
+    FloatType,
+    DoubleType,
+    StringType,
+    BinaryType,
+    DateType,
+    TimestampType,
+    DecimalType,
+    FixedType,
+    ListType,
+    MapType,
+    StructType
+)
+from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
+
+JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
+
+
+class AvroToIcebergConverter:
+    """Class to convert Avro schema to Iceberg schema."""
+
+    def __init__(self: 'AvroToIcebergConverter'):
+        self.named_type_cache: Dict[str, JsonNode] = {}
+        self.id_counter = 0
+
+    def get_id(self) -> int:
+        """Get a unique ID for a record type."""
+        self.id_counter += 1
+        return self.id_counter
+    
+    def get_fullname(self, namespace: str, name: str) -> str:
+        """Get the full name of a record type."""
+        return f"{namespace}.{name}" if namespace else name
+
+    def convert_avro_to_iceberg(self, avro_schema_path: str, avro_record_type: str, output_path: str, emit_cloudevents_columns: bool=False):
+        """Convert an Avro schema to an Iceberg schema."""
+        schema_file = avro_schema_path
+        if not schema_file:
+            print("Please specify the avro schema file")
+            sys.exit(1)
+        with open(schema_file, "r", encoding="utf-8") as f:
+            schema_json = f.read()
+
+        # Parse the schema as a JSON object
+        schema = json.loads(schema_json)
+        self.cache_named_types(schema)
+
+        if isinstance(schema, list) and avro_record_type:
+            schema = next(
+                (x for x in schema if x["name"] == avro_record_type or x["namespace"] + "." + x["name"] == avro_record_type), None)
+            if schema is None:
+                print(
+                    f"No top-level record type {avro_record_type} found in the Avro schema")
+                sys.exit(1)
+        elif not isinstance(schema, dict):
+            print(
+                "Expected a single Avro schema as a JSON object, or a list of schema records")
+            sys.exit(1)
+
+        # Get the name and fields of the top-level record
+        table_name = schema["name"]
+        fields = schema["fields"]
+
+        # Create a list to store the iceberg schema
+        iceberg_fields: List[NestedField] = []
+
+        # Append the iceberg schema with the column names and types
+        for i, field in enumerate(fields):
+            column_name = field["name"]
+            column_type = self.convert_avro_type_to_iceberg_type(field["type"])
+            iceberg_fields.append(
+                NestedField(field_id=self.get_id(), name=column_name, type=column_type))
+
+        if emit_cloudevents_columns:
+            iceberg_fields.extend([
+                NestedField(field_id=self.get_id(),
+                      name="___type", type=StringType()),
+                NestedField(field_id=self.get_id(),
+                      name="___source", type=StringType()),
+                NestedField(field_id=self.get_id(),
+                      name="___id", type=StringType()),
+                NestedField(field_id=self.get_id(),
+                      name="___time", type=TimestampType()),
+                NestedField(field_id=self.get_id(),
+                      name="___subject", type=StringType())
+            ])
+
+        iceberg_schema = Schema(*iceberg_fields)
+        arrow_schema = schema_to_pyarrow(iceberg_schema)
+        print(f"Iceberg schema created: {arrow_schema}")
+
+        # Write to Iceberg table (for demonstration, using local file system)
+        file_io = PyArrowFileIO()
+        output_file = file_io.new_output("file://"+output_path)
+        with output_file.create(overwrite=True) as f:
+            pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+
+    def convert_avro_type_to_iceberg_type(self, avro_type):
+        """Convert an Avro type to an Iceberg type."""
+        if isinstance(avro_type, list):
+            item_count = len(avro_type)
+            if item_count == 1:
+                return self.convert_avro_type_to_iceberg_type(avro_type[0])
+            elif item_count == 2:
+                first, second = avro_type[0], avro_type[1]
+                if first == "null":
+                    return self.convert_avro_type_to_iceberg_type(second)
+                elif second == "null":
+                    return self.convert_avro_type_to_iceberg_type(first)
+                else:
+                    return StructType(fields=[NestedField(field_id=self.get_id(), name=f'field_{i}', type=self.convert_avro_type_to_iceberg_type(t)) for i, t in enumerate(avro_type)])
+            elif item_count > 0:
+                return StructType(fields=[NestedField(field_id=self.get_id(), name=f'field_{i}', type=self.convert_avro_type_to_iceberg_type(t)) for i, t in enumerate(avro_type)])
+            else:
+                print(f"WARNING: Empty union type {avro_type}")
+                return StringType()
+        elif isinstance(avro_type, dict):
+            type_name = avro_type.get("type")
+            if type_name == "array":
+                return ListType(element_id=self.get_id(), element=self.convert_avro_type_to_iceberg_type(avro_type.get("items")))
+            elif type_name == "map":
+                return MapType(key_id=self.get_id(), key_type=StringType(), value_id=self.get_id(), value_type=self.convert_avro_type_to_iceberg_type(avro_type.get("values")))
+            elif type_name == "record":
+                fields = avro_type.get("fields")
+                return StructType(fields=[NestedField(field_id=self.get_id(), name=field["name"], type=self.convert_avro_type_to_iceberg_type(field["type"])) for i, field in enumerate(fields)])
+            if type_name == "enum":
+                return StringType()
+            elif type_name == "fixed":
+                return FixedType(avro_type.get("size"))
+            elif type_name == "string":
+                logical_type = avro_type.get("logicalType")
+                if logical_type == "uuid":
+                    return StringType()
+                return StringType()
+            elif type_name == "bytes":
+                logical_type = avro_type.get("logicalType")
+                if logical_type == "decimal":
+                    return DecimalType(38, 18)
+                return BinaryType()
+            elif type_name == "long":
+                logical_type = avro_type.get("logicalType")
+                if logical_type in ["timestamp-millis", "timestamp-micros"]:
+                    return TimestampType()
+                if logical_type in ["time-millis", "time-micros"]:
+                    return LongType()
+                return LongType()
+            elif type_name == "int":
+                logical_type = avro_type.get("logicalType")
+                if logical_type == "date":
+                    return DateType()
+                return IntegerType()
+            else:
+                return self.map_iceberg_scalar_type(type_name)
+        elif isinstance(avro_type, str):
+            if avro_type in self.named_type_cache:
+                return self.convert_avro_type_to_iceberg_type(self.named_type_cache[avro_type])
+            return self.map_iceberg_scalar_type(avro_type)
+
+        return StringType()
+
+    def cache_named_types(self, avro_type):
+        """Add an encountered type to the list of types."""
+        if isinstance(avro_type, list):
+            for item in avro_type:
+                self.cache_named_types(item)
+        if isinstance(avro_type, dict) and avro_type.get("name"):
+            self.named_type_cache[self.get_fullname(avro_type.get(
+                "namespace"), avro_type.get("name"))] = avro_type
+            if "fields" in avro_type:
+                for field in avro_type.get("fields"):
+                    if "type" in field:
+                        self.cache_named_types(field.get("type"))
+
+    def map_iceberg_scalar_type(self, type_name: str):
+        """Map an Avro scalar type to an Iceberg scalar type."""
+        if type_name == "null":
+            return StringType()
+        elif type_name == "int":
+            return IntegerType()
+        elif type_name == "long":
+            return LongType()
+        elif type_name == "float":
+            return FloatType()
+        elif type_name == "double":
+            return DoubleType()
+        elif type_name == "boolean":
+            return BooleanType()
+        elif type_name == "bytes":
+            return BinaryType()
+        elif type_name == "string":
+            return StringType()
+        else:
+            return StringType()
+
+
+def convert_avro_to_iceberg(avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns=False):
+    """Convert an Avro schema to an Iceberg schema."""
+    converter = AvroToIcebergConverter()
+    converter.convert_avro_to_iceberg(
+        avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns)

avrotize/avrotojava/class_test.java.jinja

@@ -0,0 +1,212 @@
+package {{ package }};
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.BeforeEach;
+import static org.junit.jupiter.api.Assertions.*;
+
+{% if imports %}
+{% for import_stmt in imports %}
+{{ import_stmt }}
+{% endfor %}
+{% endif %}
+
+/**
+ * Test class for {{ class_name }}
+ */
+public class {{ test_class_name }} {
+    private {{ class_name }} instance;
+
+    /**
+     * Set up test instance
+     */
+    @BeforeEach
+    public void setUp() {
+        instance = createInstance();
+    }
+
+    /**
+     * Create instance of {{ class_name }}
+     */
+    public static {{ class_name }} createInstance() {
+        return {{ class_name }}.createTestInstance();
+    }
+
+    /**
+     * Testing Equals and GetHashCode - positive case
+     */
+    @Test
+    public void testEqualsAndHashCode_PositiveCase() {
+        {{ class_name }} instance1 = createInstance();
+        {{ class_name }} instance2 = createInstance();
+        
+        // Test equality
+        assertEquals(instance1, instance2, "Two instances with same values should be equal");
+        
+        // Test hash codes
+        assertEquals(instance1.hashCode(), instance2.hashCode(), 
+            "Equal instances should have equal hash codes");
+    }
+
+    /**
+     * Testing Equals and GetHashCode - negative case
+     */
+    @Test
+    public void testEqualsAndHashCode_NegativeCase() {
+        {{ class_name }} instance1 = createInstance();
+        {{ class_name }} instance2 = createInstance();
+        
+        {%- for field in fields if not field.is_const %}
+        {%- if loop.first %}
+        // Modify {{ field.field_name }} and verify inequality
+        {%- set base_type = field.base_type %}
+        {%- if base_type == "String" %}
+        instance2.set{{ field.field_name | pascal }}((instance2.get{{ field.field_name | pascal }}() != null ? instance2.get{{ field.field_name | pascal }}() : "test") + "_modified");
+        {%- elif base_type == "byte[]" %}
+        byte[] modifiedBytes = instance2.get{{ field.field_name | pascal }}();
+        if (modifiedBytes != null && modifiedBytes.length > 0) {
+            modifiedBytes = java.util.Arrays.copyOf(modifiedBytes, modifiedBytes.length);
+            modifiedBytes[0] = (byte)(modifiedBytes[0] + 1);
+        } else {
+            modifiedBytes = new byte[] { 0x00, 0x01, 0x02, 0x03 };
+        }
+        instance2.set{{ field.field_name | pascal }}(modifiedBytes);
+        {%- elif base_type in ["int", "Integer", "long", "Long", "short", "Short", "byte", "Byte"] %}
+        {%- if base_type in ["Integer", "Long", "Short", "Byte"] %}
+        instance2.set{{ field.field_name | pascal }}((instance2.get{{ field.field_name | pascal }}() != null ? instance2.get{{ field.field_name | pascal }}() : 0) + 1);
+        {%- else %}
+        instance2.set{{ field.field_name | pascal }}(instance2.get{{ field.field_name | pascal }}() + 1);
+        {%- endif %}
+        {%- elif base_type in ["float", "Float", "double", "Double"] %}
+        {%- if base_type in ["Float", "Double"] %}
+        instance2.set{{ field.field_name | pascal }}((instance2.get{{ field.field_name | pascal }}() != null ? instance2.get{{ field.field_name | pascal }}() : 0.0{{ "f" if base_type == "Float" else "" }}) + 1.0{{ "f" if base_type == "Float" else "" }});
+        {%- else %}
+        instance2.set{{ field.field_name | pascal }}(instance2.get{{ field.field_name | pascal }}() + 1.0{{ "f" if base_type == "float" else "" }});
+        {%- endif %}
+        {%- elif base_type in ["boolean", "Boolean"] %}
+        {%- if base_type == "Boolean" %}
+        instance2.set{{ field.field_name | pascal }}(!(instance2.get{{ field.field_name | pascal }}() != null ? instance2.get{{ field.field_name | pascal }}() : false));
+        {%- else %}
+        instance2.set{{ field.field_name | pascal }}(!instance2.get{{ field.field_name | pascal }}());
+        {%- endif %}
+        {%- elif base_type in ["List", "Set", "ArrayList", "HashSet", "LinkedList"] %}
+        // For collection types, add a dummy element to make them different
+        {%- if base_type in ["Set", "HashSet"] %}
+        java.util.Set tempSet = new java.util.HashSet(instance2.get{{ field.field_name | pascal }}());
+        tempSet.add(new Object());
+        instance2.set{{ field.field_name | pascal }}(tempSet);
+        {%- else %}
+        java.util.List tempList = new java.util.ArrayList(instance2.get{{ field.field_name | pascal }}());
+        tempList.add(new Object());
+        instance2.set{{ field.field_name | pascal }}(tempList);
+        {%- endif %}
+        {%- elif base_type in ["Map", "HashMap", "LinkedHashMap"] %}
+        // For map types, add a dummy entry to make them different
+        java.util.Map tempMap = new java.util.HashMap(instance2.get{{ field.field_name | pascal }}());
+        tempMap.put(new Object(), new Object());
+        instance2.set{{ field.field_name | pascal }}(tempMap);
+        {%- else %}
+        // For enum or complex types, set to a different value
+        {%- if "Union" in base_type %}
+        // For union types, set to null to create a different value
+        instance2.set{{ field.field_name | pascal }}(null);
+        assertNotEquals(instance1, instance2, 
+            "Instances with different {{ field.field_name }} should not be equal");
+        {%- elif field.is_enum %}
+        // Enum - try to get a different enum value if more than one value exists
+        {%- set simple_type = base_type.split('.')[-1] if '.' in base_type else base_type %}
+        {{ simple_type }}[] enumValues = {{ simple_type }}.values();
+        if (enumValues.length > 1) {
+            instance2.set{{ field.field_name | pascal }}(enumValues[(instance2.get{{ field.field_name | pascal }}().ordinal() + 1) % enumValues.length]);
+            assertNotEquals(instance1, instance2, 
+                "Instances with different {{ field.field_name }} should not be equal");
+        }
+        // If enum has only one value, skip the inequality test for this field
+        {%- elif base_type == "Object" %}
+        // For Object type, create a new unique instance using a timestamp-based hash
+        instance2.set{{ field.field_name | pascal }}(new Object());
+        assertNotEquals(instance1, instance2, 
+            "Instances with different {{ field.field_name }} should not be equal");
+        {%- elif '.' in base_type %}
+        // For complex object types, set to null to create a different value
+        instance2.set{{ field.field_name | pascal }}(null);
+        assertNotEquals(instance1, instance2, 
+            "Instances with different {{ field.field_name }} should not be equal");
+        {%- else %}
+        instance2.set{{ field.field_name | pascal }}({{ field.test_value }});
+        assertNotEquals(instance1, instance2, 
+            "Instances with different {{ field.field_name }} should not be equal");
+        {%- endif %}
+        {%- endif %}
+        {%- endif %}
+        {%- endfor %}
+    }
+
+    {%- for field in fields %}
+    /**
+     * Testing property {{ field.field_name }}
+     */
+    @Test
+    public void test{{ field.field_name | pascal }}Property() {
+        {%- if '<' in field.field_type %}
+        {{ field.field_type }} testValue = {{ field.test_value }};
+        {%- else %}
+        {%- set simple_field_type = field.field_type.split('.')[-1] if '.' in field.field_type else field.field_type %}
+        {{ simple_field_type }} testValue = {{ field.test_value }};
+        {%- endif %}
+        {%- if not field.is_const %}
+        instance.set{{ field.field_name | pascal }}(testValue);
+        {%- endif %}
+        {%- if field.is_discriminator %}
+        assertEquals(testValue, instance.get{{ field.field_name | pascal }}Value());
+        {%- else %}
+        assertEquals(testValue, instance.get{{ field.field_name | pascal }}());
+        {%- endif %}
+    }
+    {%- endfor %}
+    {%- if avro_annotation %}
+    /**
+     * Testing Avro serializer
+     */
+    @Test
+    public void test_ToByteArray_FromData_Avro() throws Exception {
+        String mediaType = "application/vnd.apache.avro+avro";
+        byte[] bytes = instance.toByteArray(mediaType);
+        {{ class_name }} newInstance = {{ class_name }}.fromData(bytes, mediaType);
+        assertEquals(instance, newInstance);
+    }
+
+    /**
+     * Testing Avro serializer with gzip compression
+     */
+    @Test
+    public void test_ToByteArray_FromData_Avro_Gzip() throws Exception {
+        String mediaType = "application/vnd.apache.avro+avro+gzip";
+        byte[] bytes = instance.toByteArray(mediaType);
+        {{ class_name }} newInstance = {{ class_name }}.fromData(bytes, mediaType);
+        assertEquals(instance, newInstance);
+    }
+    {%- endif %}
+    {%- if jackson_annotation %}
+    /**
+     * Testing JSON serializer
+     */
+    @Test
+    public void test_ToByteArray_FromData_Json() throws Exception {
+        String mediaType = "application/json";
+        byte[] bytes = instance.toByteArray(mediaType);
+        {{ class_name }} newInstance = {{ class_name }}.fromData(bytes, mediaType);
+        assertEquals(instance, newInstance);
+    }
+
+    /**
+     * Testing JSON serializer with gzip compression
+     */
+    @Test
+    public void test_ToByteArray_FromData_Json_Gzip() throws Exception {
+        String mediaType = "application/json+gzip";
+        byte[] bytes = instance.toByteArray(mediaType);
+        {{ class_name }} newInstance = {{ class_name }}.fromData(bytes, mediaType);
+        assertEquals(instance, newInstance);
+    }
+    {%- endif %}
+}

avrotize/avrotojava/enum_test.java.jinja

@@ -0,0 +1,21 @@
+package {{ package }};
+
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Test class for {{ enum_name }}
+ */
+public class {{ test_class_name }} {
+    /**
+     * Test {{ enum_name }} Enum
+     */
+    @Test
+    public void test_{{ enum_name }}_Enum() {
+        {{ enum_name }}[] values = {{ enum_name }}.values();
+        {%- for symbol in symbols %}
+        assertNotNull({{ enum_name }}.valueOf("{{ symbol }}"));
+        {%- endfor %}
+        assertEquals({{ symbols|length }}, values.length);
+    }
+}

avrotize/avrotojava/testproject.pom.jinja

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>{{ groupid }}</groupId>
+    <artifactId>{{ artifactid }}-test</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <properties>
+        <maven.compiler.source>{{ JDK_VERSION }}</maven.compiler.source>
+        <maven.compiler.target>{{ JDK_VERSION }}</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>{{ groupid }}</groupId>
+            <artifactId>{{ artifactid }}</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>{{ JUNIT_VERSION }}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>{{ JUNIT_VERSION }}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+            <version>{{ AVRO_VERSION }}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson</groupId>
+            <artifactId>jackson-bom</artifactId>
+            <version>{{ JACKSON_VERSION }}</version>
+            <type>pom</type>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>{{ MAVEN_SUREFIRE_VERSION }}</version>
+            </plugin>
+        </plugins>
+    </build>
+</project>
+