avrotize 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

avrotize/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.0.1'
-__version_tuple__ = version_tuple = (3, 0, 1)
+__version__ = version = '3.1.0'
+__version_tuple__ = version_tuple = (3, 1, 0)
 
 __commit_id__ = commit_id = None

avrotize/avrotize.py

@@ -38,6 +38,10 @@ def create_subparsers(subparsers, commands):
                 kwargs['choices'] = arg['choices']
             if 'default' in arg:
                 kwargs['default'] = arg['default']
+            # Handle dest for optional arguments only (positional args can't have dest)
+            arg_is_positional = not arg['name'].startswith('-')
+            if 'dest' in arg and not arg_is_positional:
+                kwargs['dest'] = arg['dest']
             if arg['type'] == 'bool':
                 kwargs['action'] = 'store_true'
                 del kwargs['type']

avrotize/avrotoiceberg.py

@@ -5,6 +5,7 @@ import sys
 from typing import Dict, List
 import pyarrow as pa
 from pyiceberg.schema import Schema, NestedField
+from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 from pyiceberg.types import (
     BooleanType,
     IntegerType,
@@ -21,11 +22,74 @@ from pyiceberg.types import (
     MapType,
     StructType
 )
-from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 
 JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
 
 
+def iceberg_type_to_json(iceberg_type) -> str | Dict:
+    """
+    Serialize an Iceberg type to JSON per Iceberg Table Spec Appendix C.
+    
+    Primitive types are serialized as strings. Complex types (struct, list, map)
+    are serialized as JSON objects with their nested structure.
+    """
+    # Primitive types map to simple strings
+    if isinstance(iceberg_type, BooleanType):
+        return "boolean"
+    elif isinstance(iceberg_type, IntegerType):
+        return "int"
+    elif isinstance(iceberg_type, LongType):
+        return "long"
+    elif isinstance(iceberg_type, FloatType):
+        return "float"
+    elif isinstance(iceberg_type, DoubleType):
+        return "double"
+    elif isinstance(iceberg_type, StringType):
+        return "string"
+    elif isinstance(iceberg_type, BinaryType):
+        return "binary"
+    elif isinstance(iceberg_type, DateType):
+        return "date"
+    elif isinstance(iceberg_type, TimestampType):
+        return "timestamp"
+    elif isinstance(iceberg_type, DecimalType):
+        return f"decimal({iceberg_type.precision},{iceberg_type.scale})"
+    elif isinstance(iceberg_type, FixedType):
+        return f"fixed[{iceberg_type.length}]"
+    elif isinstance(iceberg_type, ListType):
+        return {
+            "type": "list",
+            "element-id": iceberg_type.element_id,
+            "element-required": iceberg_type.element_required,
+            "element": iceberg_type_to_json(iceberg_type.element_type)
+        }
+    elif isinstance(iceberg_type, MapType):
+        return {
+            "type": "map",
+            "key-id": iceberg_type.key_id,
+            "key": iceberg_type_to_json(iceberg_type.key_type),
+            "value-id": iceberg_type.value_id,
+            "value-required": iceberg_type.value_required,
+            "value": iceberg_type_to_json(iceberg_type.value_type)
+        }
+    elif isinstance(iceberg_type, StructType):
+        return {
+            "type": "struct",
+            "fields": [
+                {
+                    "id": field.field_id,
+                    "name": field.name,
+                    "required": field.required,
+                    "type": iceberg_type_to_json(field.field_type)
+                }
+                for field in iceberg_type.fields
+            ]
+        }
+    else:
+        # Fallback for unknown types
+        return str(iceberg_type)
+
+
 class AvroToIcebergConverter:
     """Class to convert Avro schema to Iceberg schema."""
 
@@ -42,8 +106,16 @@ class AvroToIcebergConverter:
         """Get the full name of a record type."""
         return f"{namespace}.{name}" if namespace else name
 
-    def convert_avro_to_iceberg(self, avro_schema_path: str, avro_record_type: str, output_path: str, emit_cloudevents_columns: bool=False):
-        """Convert an Avro schema to an Iceberg schema."""
+    def convert_avro_to_iceberg(self, avro_schema_path: str, avro_record_type: str, output_path: str, emit_cloudevents_columns: bool=False, output_format: str="arrow"):
+        """Convert an Avro schema to an Iceberg schema.
+        
+        Args:
+            avro_schema_path: Path to the Avro schema file
+            avro_record_type: Record type to convert (or None for the root)
+            output_path: Path to write the Iceberg schema
+            emit_cloudevents_columns: Whether to add CloudEvents columns
+            output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+        """
         schema_file = avro_schema_path
         if not schema_file:
             print("Please specify the avro schema file")
@@ -96,14 +168,32 @@ class AvroToIcebergConverter:
             ])
 
         iceberg_schema = Schema(*iceberg_fields)
-        arrow_schema = schema_to_pyarrow(iceberg_schema)
-        print(f"Iceberg schema created: {arrow_schema}")
+        print(f"Iceberg schema created: {iceberg_schema}")
 
-        # Write to Iceberg table (for demonstration, using local file system)
-        file_io = PyArrowFileIO()
-        output_file = file_io.new_output("file://"+output_path)
-        with output_file.create(overwrite=True) as f:
-            pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        if output_format == "arrow":
+            # Write as binary PyArrow schema
+            arrow_schema = schema_to_pyarrow(iceberg_schema)
+            file_io = PyArrowFileIO()
+            output_file = file_io.new_output("file://" + output_path)
+            with output_file.create(overwrite=True) as f:
+                pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        else:
+            # Write Iceberg schema as spec-compliant JSON (per Iceberg Table Spec Appendix C)
+            schema_json = {
+                "type": "struct",
+                "schema-id": 0,
+                "fields": [
+                    {
+                        "id": field.field_id,
+                        "name": field.name,
+                        "required": field.required,
+                        "type": iceberg_type_to_json(field.field_type)
+                    }
+                    for field in iceberg_schema.fields
+                ]
+            }
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump(schema_json, f, indent=2)
 
     def convert_avro_type_to_iceberg_type(self, avro_type):
         """Convert an Avro type to an Iceberg type."""
@@ -203,8 +293,16 @@ class AvroToIcebergConverter:
             return StringType()
 
 
-def convert_avro_to_iceberg(avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns=False):
-    """Convert an Avro schema to an Iceberg schema."""
+def convert_avro_to_iceberg(avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns=False, output_format="arrow"):
+    """Convert an Avro schema to an Iceberg schema.
+    
+    Args:
+        avro_schema_path: Path to the Avro schema file
+        avro_record_type: Record type to convert (or None for the root)
+        output_path: Path to write the Iceberg schema
+        emit_cloudevents_columns: Whether to add CloudEvents columns
+        output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+    """
     converter = AvroToIcebergConverter()
     converter.convert_avro_to_iceberg(
-        avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns)
+        avro_schema_path, avro_record_type, output_path, emit_cloudevents_columns, output_format)

avrotize/avrotots/class_core.ts.jinja

@@ -8,7 +8,7 @@ import { jsonObject, jsonMember, TypedJSON } from 'typedjson';
 {%- endif %}
 {%- endif %}
 {%- if avro_annotation %}
-import avro from 'avro-js';
+import avro, { type Type } from 'avro-js';
 {%- endif %}
 {%- for import_type, import_path in imports.items() %}
 import { {{ import_type }} } from '{{ import_path }}';
@@ -22,7 +22,7 @@ import pako from 'pako';
 {%- endif %}
 export class {{ class_name }} {
     {%- if avro_annotation %}
-    public static AvroType: avro.Type = avro.parse({{ avro_schema_json }});
+    public static AvroType: Type = avro.parse({{ avro_schema_json }});
     {%- endif %}
 
     {%- for field in fields %}

avrotize/avrotots.py

@@ -500,10 +500,11 @@ class AvroToTypeScript:
         """Generate TypeScript type declaration file for avro-js module."""
         avro_js_types = '''declare module 'avro-js' {
     /**
-     * Avro Type representation.
+     * Avro Type interface.
+     * Represents the structure of Type instances returned by avro.parse().
      * Provides methods for encoding, decoding, and validating Avro data.
      */
-    export class Type {
+    export interface Type {
         /**
          * Encode a value to a Buffer.
          * @param obj - Value to encode
@@ -575,12 +576,66 @@ class AvroToTypeScript:
     }
 
     /**
-     * Parse an Avro schema and return a Type instance.
-     * @param schema - Schema as string or object
-     * @param options - Parse options
-     * @returns Type instance
+     * avro-js default export interface.
+     * This module is CommonJS, so in ESM context it only has a default export.
      */
-    export function parse(schema: string | any, options?: any): Type;
+    export interface Avro {
+        /**
+         * Type class constructor.
+         */
+        Type: any;
+
+        /**
+         * Parse an Avro schema and return a Type instance.
+         * @param schema - Schema as string or object
+         * @param options - Parse options
+         * @returns Type instance
+         */
+        parse(schema: string | any, options?: any): Type;
+
+        /**
+         * Protocol class constructor.
+         */
+        Protocol: any;
+
+        /**
+         * Create a file decoder.
+         */
+        createFileDecoder(path: string, options?: any): any;
+
+        /**
+         * Create a file encoder.
+         */
+        createFileEncoder(path: string, schema: any, options?: any): any;
+
+        /**
+         * Extract file header.
+         */
+        extractFileHeader(buffer: Buffer): any;
+
+        /**
+         * Streams utilities.
+         */
+        streams: any;
+
+        /**
+         * Built-in types.
+         */
+        types: any;
+
+        /**
+         * Validator (deprecated).
+         */
+        Validator: any;
+
+        /**
+         * ProtocolValidator (deprecated).
+         */
+        ProtocolValidator: any;
+    }
+
+    const avro: Avro;
+    export default avro;
 }
 '''