avrotize 3.0.1__tar.gz → 3.0.2__tar.gz

{avrotize-3.0.1 → avrotize-3.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avrotize
-Version: 3.0.1
+Version: 3.0.2
 Summary: Tools to convert from and to Avro Schema from various other schema languages.
 Author-email: Clemens Vasters <clemensv@microsoft.com>
 Requires-Python: >=3.10
@@ -676,7 +676,7 @@ Conversion notes:
 ### Convert Avrotize Schema to Iceberg schema
 
 ```bash
-avrotize a2ib <path_to_avro_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-avro>] [--emit-cloudevents-columns]
+avrotize a2ib <path_to_avro_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-avro>] [--emit-cloudevents-columns] [--format schema|arrow]
 ```
 
 Parameters:
@@ -685,6 +685,7 @@ Parameters:
 - `--out`: The path to the Iceberg schema file to write the conversion result to. If omitted, the output is directed to stdout.
 - `--record-type`: (optional) The name of the Avro record type to convert to an Iceberg schema.
 - `--emit-cloudevents-columns`: (optional) If set, the tool will add [CloudEvents](https://cloudevents.io) attribute columns to the Iceberg schema: `__id`, `__source`, `__subject`, `__type`, and `__time`.
+- `--format`: (optional) Output format. `schema` (default) outputs JSON per the [Iceberg Table Spec Appendix C](https://iceberg.apache.org/spec/#appendix-c-json-serialization). `arrow` outputs a binary Arrow IPC serialized schema.
 
 Conversion notes:
 
@@ -695,7 +696,7 @@ Conversion notes:
 ### Convert JSON Structure to Iceberg schema
 
 ```bash
-avrotize s2ib <path_to_structure_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-structure>] [--emit-cloudevents-columns]
+avrotize s2ib <path_to_structure_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-structure>] [--emit-cloudevents-columns] [--format schema|arrow]
 ```
 
 Parameters:
@@ -704,6 +705,7 @@ Parameters:
 - `--out`: The path to the Iceberg schema file to write the conversion result to. If omitted, the output is directed to stdout.
 - `--record-type`: (optional) The name of the record type in definitions to convert to an Iceberg schema.
 - `--emit-cloudevents-columns`: (optional) If set, the tool will add [CloudEvents](https://cloudevents.io) attribute columns to the Iceberg schema: `___id`, `___source`, `___subject`, `___type`, and `___time`.
+- `--format`: (optional) Output format. `schema` (default) outputs JSON per the [Iceberg Table Spec Appendix C](https://iceberg.apache.org/spec/#appendix-c-json-serialization). `arrow` outputs a binary Arrow IPC serialized schema.
 
 Conversion notes:
 

{avrotize-3.0.1 → avrotize-3.0.2}/README.md

@@ -628,7 +628,7 @@ Conversion notes:
 ### Convert Avrotize Schema to Iceberg schema
 
 ```bash
-avrotize a2ib <path_to_avro_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-avro>] [--emit-cloudevents-columns]
+avrotize a2ib <path_to_avro_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-avro>] [--emit-cloudevents-columns] [--format schema|arrow]
 ```
 
 Parameters:
@@ -637,6 +637,7 @@ Parameters:
 - `--out`: The path to the Iceberg schema file to write the conversion result to. If omitted, the output is directed to stdout.
 - `--record-type`: (optional) The name of the Avro record type to convert to an Iceberg schema.
 - `--emit-cloudevents-columns`: (optional) If set, the tool will add [CloudEvents](https://cloudevents.io) attribute columns to the Iceberg schema: `__id`, `__source`, `__subject`, `__type`, and `__time`.
+- `--format`: (optional) Output format. `schema` (default) outputs JSON per the [Iceberg Table Spec Appendix C](https://iceberg.apache.org/spec/#appendix-c-json-serialization). `arrow` outputs a binary Arrow IPC serialized schema.
 
 Conversion notes:
 
@@ -647,7 +648,7 @@ Conversion notes:
 ### Convert JSON Structure to Iceberg schema
 
 ```bash
-avrotize s2ib <path_to_structure_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-structure>] [--emit-cloudevents-columns]
+avrotize s2ib <path_to_structure_schema_file> [--out <path_to_iceberg_schema_file>] [--record-type <record-type-from-structure>] [--emit-cloudevents-columns] [--format schema|arrow]
 ```
 
 Parameters:
@@ -656,6 +657,7 @@ Parameters:
 - `--out`: The path to the Iceberg schema file to write the conversion result to. If omitted, the output is directed to stdout.
 - `--record-type`: (optional) The name of the record type in definitions to convert to an Iceberg schema.
 - `--emit-cloudevents-columns`: (optional) If set, the tool will add [CloudEvents](https://cloudevents.io) attribute columns to the Iceberg schema: `___id`, `___source`, `___subject`, `___type`, and `___time`.
+- `--format`: (optional) Output format. `schema` (default) outputs JSON per the [Iceberg Table Spec Appendix C](https://iceberg.apache.org/spec/#appendix-c-json-serialization). `arrow` outputs a binary Arrow IPC serialized schema.
 
 Conversion notes:
 

{avrotize-3.0.1 → avrotize-3.0.2}/avrotize/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.0.1'
-__version_tuple__ = version_tuple = (3, 0, 1)
+__version__ = version = '3.0.2'
+__version_tuple__ = version_tuple = (3, 0, 2)
 
-__commit_id__ = commit_id = 'g62e6bb75a'
+__commit_id__ = commit_id = 'g3ee5f8f67'

{avrotize-3.0.1 → avrotize-3.0.2}/avrotize/avrotoiceberg.py

@@ -5,6 +5,7 @@ import sys
 from typing import Dict, List
 import pyarrow as pa
 from pyiceberg.schema import Schema, NestedField
+from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 from pyiceberg.types import (
     BooleanType,
     IntegerType,
@@ -21,11 +22,74 @@ from pyiceberg.types import (
     MapType,
     StructType
 )
-from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 
 JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
 
 
+def iceberg_type_to_json(iceberg_type) -> str | Dict:
+    """
+    Serialize an Iceberg type to JSON per Iceberg Table Spec Appendix C.
+    
+    Primitive types are serialized as strings. Complex types (struct, list, map)
+    are serialized as JSON objects with their nested structure.
+    """
+    # Primitive types map to simple strings
+    if isinstance(iceberg_type, BooleanType):
+        return "boolean"
+    elif isinstance(iceberg_type, IntegerType):
+        return "int"
+    elif isinstance(iceberg_type, LongType):
+        return "long"
+    elif isinstance(iceberg_type, FloatType):
+        return "float"
+    elif isinstance(iceberg_type, DoubleType):
+        return "double"
+    elif isinstance(iceberg_type, StringType):
+        return "string"
+    elif isinstance(iceberg_type, BinaryType):
+        return "binary"
+    elif isinstance(iceberg_type, DateType):
+        return "date"
+    elif isinstance(iceberg_type, TimestampType):
+        return "timestamp"
+    elif isinstance(iceberg_type, DecimalType):
+        return f"decimal({iceberg_type.precision},{iceberg_type.scale})"
+    elif isinstance(iceberg_type, FixedType):
+        return f"fixed[{iceberg_type.length}]"
+    elif isinstance(iceberg_type, ListType):
+        return {
+            "type": "list",
+            "element-id": iceberg_type.element_id,
+            "element-required": iceberg_type.element_required,
+            "element": iceberg_type_to_json(iceberg_type.element_type)
+        }
+    elif isinstance(iceberg_type, MapType):
+        return {
+            "type": "map",
+            "key-id": iceberg_type.key_id,
+            "key": iceberg_type_to_json(iceberg_type.key_type),
+            "value-id": iceberg_type.value_id,
+            "value-required": iceberg_type.value_required,
+            "value": iceberg_type_to_json(iceberg_type.value_type)
+        }
+    elif isinstance(iceberg_type, StructType):
+        return {
+            "type": "struct",
+            "fields": [
+                {
+                    "id": field.field_id,
+                    "name": field.name,
+                    "required": field.required,
+                    "type": iceberg_type_to_json(field.field_type)
+                }
+                for field in iceberg_type.fields
+            ]
+        }
+    else:
+        # Fallback for unknown types
+        return str(iceberg_type)
+
+
 class AvroToIcebergConverter:
     """Class to convert Avro schema to Iceberg schema."""
 
@@ -42,8 +106,16 @@ class AvroToIcebergConverter:
         """Get the full name of a record type."""
         return f"{namespace}.{name}" if namespace else name
 
-    def convert_avro_to_iceberg(self, avro_schema_path: str, avro_record_type: str, output_path: str, emit_cloudevents_columns: bool=False):
-        """Convert an Avro schema to an Iceberg schema."""
+    def convert_avro_to_iceberg(self, avro_schema_path: str, avro_record_type: str, output_path: str, emit_cloudevents_columns: bool=False, output_format: str="arrow"):
+        """Convert an Avro schema to an Iceberg schema.
+        
+        Args:
+            avro_schema_path: Path to the Avro schema file
+            avro_record_type: Record type to convert (or None for the root)
+            output_path: Path to write the Iceberg schema
+            emit_cloudevents_columns: Whether to add CloudEvents columns
+            output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+        """
         schema_file = avro_schema_path
         if not schema_file:
             print("Please specify the avro schema file")
@@ -96,14 +168,32 @@ class AvroToIcebergConverter:
             ])
 
         iceberg_schema = Schema(*iceberg_fields)
-        arrow_schema = schema_to_pyarrow(iceberg_schema)
-        print(f"Iceberg schema created: {arrow_schema}")
+        print(f"Iceberg schema created: {iceberg_schema}")
 
-        # Write to Iceberg table (for demonstration, using local file system)
-        file_io = PyArrowFileIO()
-        output_file = file_io.new_output("file://"+output_path)
-        with output_file.create(overwrite=True) as f:
-            pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        if output_format == "arrow":
+            # Write as binary PyArrow schema
+            arrow_schema = schema_to_pyarrow(iceberg_schema)
+            file_io = PyArrowFileIO()
+            output_file = file_io.new_output("file://" + output_path)
+            with output_file.create(overwrite=True) as f:
+                pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        else:
+            # Write Iceberg schema as spec-compliant JSON (per Iceberg Table Spec Appendix C)
+            schema_json = {
+                "type": "struct",
+                "schema-id": 0,
+                "fields": [
+                    {
+                        "id": field.field_id,
+                        "name": field.name,
+                        "required": field.required,
+                        "type": iceberg_type_to_json(field.field_type)
+                    }
+                    for field in iceberg_schema.fields
+                ]
+            }
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump(schema_json, f, indent=2)
 
     def convert_avro_type_to_iceberg_type(self, avro_type):
         """Convert an Avro type to an Iceberg type."""
@@ -203,8 +293,16 @@ class AvroToIcebergConverter:
             return StringType()
 
 
-def convert_avro_to_iceberg(avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns=False):
-    """Convert an Avro schema to an Iceberg schema."""
+def convert_avro_to_iceberg(avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns=False, output_format="arrow"):
+    """Convert an Avro schema to an Iceberg schema.
+    
+    Args:
+        avro_schema_path: Path to the Avro schema file
+        avro_record_type: Record type to convert (or None for the root)
+        output_path: Path to write the Iceberg schema
+        emit_cloudevents_columns: Whether to add CloudEvents columns
+        output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+    """
     converter = AvroToIcebergConverter()
     converter.convert_avro_to_iceberg(
-        avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns)
+        avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns, output_format)

{avrotize-3.0.1 → avrotize-3.0.2}/avrotize/commands.json

@@ -1115,7 +1115,8 @@
         "avro_schema_path": "input_file_path",
         "output_path": "output_file_path",
         "avro_record_type": "args.record_type",
-        "emit_cloudevents_columns": "args.emit_cloudevents_columns"
+        "emit_cloudevents_columns": "args.emit_cloudevents_columns",
+        "output_format": "args.format"
       }
     },
     "extensions": [
@@ -1147,6 +1148,14 @@
         "help": "Add CloudEvents columns to the Iceberg schema",
         "default": false,
         "required": false
+      },
+      {
+        "name": "--format",
+        "type": "str",
+        "help": "Output format: 'arrow' for binary Arrow IPC (default), 'schema' for JSON",
+        "choices": ["schema", "arrow"],
+        "default": "arrow",
+        "required": false
       }
     ],
     "suggested_output_file_path": "{input_file_name}.iceberg",
@@ -1169,7 +1178,8 @@
         "structure_schema_path": "input_file_path",
         "output_path": "output_file_path",
         "structure_record_type": "args.record_type",
-        "emit_cloudevents_columns": "args.emit_cloudevents_columns"
+        "emit_cloudevents_columns": "args.emit_cloudevents_columns",
+        "output_format": "args.format"
       }
     },
     "extensions": [
@@ -1202,6 +1212,14 @@
         "help": "Add CloudEvents columns to the Iceberg schema",
         "default": false,
         "required": false
+      },
+      {
+        "name": "--format",
+        "type": "str",
+        "help": "Output format: 'arrow' for binary Arrow IPC (default), 'schema' for JSON",
+        "choices": ["schema", "arrow"],
+        "default": "arrow",
+        "required": false
       }
     ],
     "suggested_output_file_path": "{input_file_name}.iceberg",

{avrotize-3.0.1 → avrotize-3.0.2}/avrotize/structuretodb.py

@@ -426,7 +426,7 @@ def structure_type_to_sql_type(structure_type: Any, dialect: str) -> str:
     type_map["sqlanywhere"] = type_map["sqlserver"].copy()
     type_map["bigquery"] = {k: v.replace("VARCHAR", "STRING").replace("JSONB", "STRING").replace("BYTEA", "BYTES") for k, v in type_map["postgres"].items()}
     type_map["snowflake"] = {k: v.replace("JSONB", "VARIANT").replace("BYTEA", "BINARY") for k, v in type_map["postgres"].items()}
-    type_map["redshift"] = type_map["postgres"].copy()
+    type_map["redshift"] = {k: v.replace("JSONB", "SUPER").replace("BYTEA", "VARBYTE") for k, v in type_map["postgres"].items()}
 
     # Handle type resolution
     if isinstance(structure_type, str):

{avrotize-3.0.1 → avrotize-3.0.2}/avrotize/structuretoiceberg.py

@@ -5,6 +5,7 @@ import sys
 from typing import Dict, List, Any, Optional
 import pyarrow as pa
 from pyiceberg.schema import Schema, NestedField
+from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 from pyiceberg.types import (
     BooleanType,
     IntegerType,
@@ -22,11 +23,76 @@ from pyiceberg.types import (
     StructType,
     TimeType
 )
-from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 
 JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
 
 
+def iceberg_type_to_json(iceberg_type) -> str | Dict:
+    """
+    Serialize an Iceberg type to JSON per Iceberg Table Spec Appendix C.
+    
+    Primitive types are serialized as strings. Complex types (struct, list, map)
+    are serialized as JSON objects with their nested structure.
+    """
+    # Primitive types map to simple strings
+    if isinstance(iceberg_type, BooleanType):
+        return "boolean"
+    elif isinstance(iceberg_type, IntegerType):
+        return "int"
+    elif isinstance(iceberg_type, LongType):
+        return "long"
+    elif isinstance(iceberg_type, FloatType):
+        return "float"
+    elif isinstance(iceberg_type, DoubleType):
+        return "double"
+    elif isinstance(iceberg_type, StringType):
+        return "string"
+    elif isinstance(iceberg_type, BinaryType):
+        return "binary"
+    elif isinstance(iceberg_type, DateType):
+        return "date"
+    elif isinstance(iceberg_type, TimeType):
+        return "time"
+    elif isinstance(iceberg_type, TimestampType):
+        return "timestamp"
+    elif isinstance(iceberg_type, DecimalType):
+        return f"decimal({iceberg_type.precision},{iceberg_type.scale})"
+    elif isinstance(iceberg_type, FixedType):
+        return f"fixed[{iceberg_type.length}]"
+    elif isinstance(iceberg_type, ListType):
+        return {
+            "type": "list",
+            "element-id": iceberg_type.element_id,
+            "element-required": iceberg_type.element_required,
+            "element": iceberg_type_to_json(iceberg_type.element_type)
+        }
+    elif isinstance(iceberg_type, MapType):
+        return {
+            "type": "map",
+            "key-id": iceberg_type.key_id,
+            "key": iceberg_type_to_json(iceberg_type.key_type),
+            "value-id": iceberg_type.value_id,
+            "value-required": iceberg_type.value_required,
+            "value": iceberg_type_to_json(iceberg_type.value_type)
+        }
+    elif isinstance(iceberg_type, StructType):
+        return {
+            "type": "struct",
+            "fields": [
+                {
+                    "id": field.field_id,
+                    "name": field.name,
+                    "required": field.required,
+                    "type": iceberg_type_to_json(field.field_type)
+                }
+                for field in iceberg_type.fields
+            ]
+        }
+    else:
+        # Fallback for unknown types
+        return str(iceberg_type)
+
+
 class StructureToIcebergConverter:
     """Class to convert JSON Structure schema to Iceberg schema."""
 
@@ -45,8 +111,16 @@ class StructureToIcebergConverter:
         """Get the full name of a record type."""
         return f"{namespace}.{name}" if namespace else name
 
-    def convert_structure_to_iceberg(self, structure_schema_path: str, structure_record_type: Optional[str], output_path: str, emit_cloudevents_columns: bool=False):
-        """Convert a JSON Structure schema to an Iceberg schema."""
+    def convert_structure_to_iceberg(self, structure_schema_path: str, structure_record_type: Optional[str], output_path: str, emit_cloudevents_columns: bool=False, output_format: str="arrow"):
+        """Convert a JSON Structure schema to an Iceberg schema.
+        
+        Args:
+            structure_schema_path: Path to the JSON Structure schema file
+            structure_record_type: Record type to convert (or None for the root)
+            output_path: Path to write the Iceberg schema
+            emit_cloudevents_columns: Whether to add CloudEvents columns
+            output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+        """
         schema_file = structure_schema_path
         if not schema_file:
             print("Please specify the JSON Structure schema file")
@@ -114,14 +188,32 @@ class StructureToIcebergConverter:
             ])
 
         iceberg_schema = Schema(*iceberg_fields)
-        arrow_schema = schema_to_pyarrow(iceberg_schema)
-        print(f"Iceberg schema created: {arrow_schema}")
+        print(f"Iceberg schema created: {iceberg_schema}")
 
-        # Write to Iceberg table (for demonstration, using local file system)
-        file_io = PyArrowFileIO()
-        output_file = file_io.new_output("file://"+output_path)
-        with output_file.create(overwrite=True) as f:
-            pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        if output_format == "arrow":
+            # Write as binary PyArrow schema
+            arrow_schema = schema_to_pyarrow(iceberg_schema)
+            file_io = PyArrowFileIO()
+            output_file = file_io.new_output("file://" + output_path)
+            with output_file.create(overwrite=True) as f:
+                pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        else:
+            # Write Iceberg schema as spec-compliant JSON (per Iceberg Table Spec Appendix C)
+            schema_json = {
+                "type": "struct",
+                "schema-id": 0,
+                "fields": [
+                    {
+                        "id": field.field_id,
+                        "name": field.name,
+                        "required": field.required,
+                        "type": iceberg_type_to_json(field.field_type)
+                    }
+                    for field in iceberg_schema.fields
+                ]
+            }
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump(schema_json, f, indent=2)
 
     def resolve_ref(self, ref: str) -> Dict[str, Any]:
         """Resolve a $ref reference."""
@@ -348,8 +440,16 @@ class StructureToIcebergConverter:
         return type_mapping.get(type_name, StringType())
 
 
-def convert_structure_to_iceberg(structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns=False):
-    """Convert a JSON Structure schema to an Iceberg schema."""
+def convert_structure_to_iceberg(structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns=False, output_format="arrow"):
+    """Convert a JSON Structure schema to an Iceberg schema.
+    
+    Args:
+        structure_schema_path: Path to the JSON Structure schema file
+        structure_record_type: Record type to convert (or None for the root)
+        output_path: Path to write the Iceberg schema
+        emit_cloudevents_columns: Whether to add CloudEvents columns
+        output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+    """
     converter = StructureToIcebergConverter()
     converter.convert_structure_to_iceberg(
-        structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns)
+        structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns, output_format)