avrotize 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

avrotize/structuretodb.py

@@ -426,7 +426,7 @@ def structure_type_to_sql_type(structure_type: Any, dialect: str) -> str:
     type_map["sqlanywhere"] = type_map["sqlserver"].copy()
     type_map["bigquery"] = {k: v.replace("VARCHAR", "STRING").replace("JSONB", "STRING").replace("BYTEA", "BYTES") for k, v in type_map["postgres"].items()}
     type_map["snowflake"] = {k: v.replace("JSONB", "VARIANT").replace("BYTEA", "BINARY") for k, v in type_map["postgres"].items()}
-    type_map["redshift"] = type_map["postgres"].copy()
+    type_map["redshift"] = {k: v.replace("JSONB", "SUPER").replace("BYTEA", "VARBYTE") for k, v in type_map["postgres"].items()}
 
     # Handle type resolution
     if isinstance(structure_type, str):

avrotize/structuretoiceberg.py

@@ -5,6 +5,7 @@ import sys
 from typing import Dict, List, Any, Optional
 import pyarrow as pa
 from pyiceberg.schema import Schema, NestedField
+from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 from pyiceberg.types import (
     BooleanType,
     IntegerType,
@@ -22,11 +23,76 @@ from pyiceberg.types import (
     StructType,
     TimeType
 )
-from pyiceberg.io.pyarrow import PyArrowFileIO, schema_to_pyarrow
 
 JsonNode = Dict[str, 'JsonNode'] | List['JsonNode'] | str | bool | int | None
 
 
+def iceberg_type_to_json(iceberg_type) -> str | Dict:
+    """
+    Serialize an Iceberg type to JSON per Iceberg Table Spec Appendix C.
+    
+    Primitive types are serialized as strings. Complex types (struct, list, map)
+    are serialized as JSON objects with their nested structure.
+    """
+    # Primitive types map to simple strings
+    if isinstance(iceberg_type, BooleanType):
+        return "boolean"
+    elif isinstance(iceberg_type, IntegerType):
+        return "int"
+    elif isinstance(iceberg_type, LongType):
+        return "long"
+    elif isinstance(iceberg_type, FloatType):
+        return "float"
+    elif isinstance(iceberg_type, DoubleType):
+        return "double"
+    elif isinstance(iceberg_type, StringType):
+        return "string"
+    elif isinstance(iceberg_type, BinaryType):
+        return "binary"
+    elif isinstance(iceberg_type, DateType):
+        return "date"
+    elif isinstance(iceberg_type, TimeType):
+        return "time"
+    elif isinstance(iceberg_type, TimestampType):
+        return "timestamp"
+    elif isinstance(iceberg_type, DecimalType):
+        return f"decimal({iceberg_type.precision},{iceberg_type.scale})"
+    elif isinstance(iceberg_type, FixedType):
+        return f"fixed[{iceberg_type.length}]"
+    elif isinstance(iceberg_type, ListType):
+        return {
+            "type": "list",
+            "element-id": iceberg_type.element_id,
+            "element-required": iceberg_type.element_required,
+            "element": iceberg_type_to_json(iceberg_type.element_type)
+        }
+    elif isinstance(iceberg_type, MapType):
+        return {
+            "type": "map",
+            "key-id": iceberg_type.key_id,
+            "key": iceberg_type_to_json(iceberg_type.key_type),
+            "value-id": iceberg_type.value_id,
+            "value-required": iceberg_type.value_required,
+            "value": iceberg_type_to_json(iceberg_type.value_type)
+        }
+    elif isinstance(iceberg_type, StructType):
+        return {
+            "type": "struct",
+            "fields": [
+                {
+                    "id": field.field_id,
+                    "name": field.name,
+                    "required": field.required,
+                    "type": iceberg_type_to_json(field.field_type)
+                }
+                for field in iceberg_type.fields
+            ]
+        }
+    else:
+        # Fallback for unknown types
+        return str(iceberg_type)
+
+
 class StructureToIcebergConverter:
     """Class to convert JSON Structure schema to Iceberg schema."""
 
@@ -45,8 +111,16 @@ class StructureToIcebergConverter:
         """Get the full name of a record type."""
         return f"{namespace}.{name}" if namespace else name
 
-    def convert_structure_to_iceberg(self, structure_schema_path: str, structure_record_type: Optional[str], output_path: str, emit_cloudevents_columns: bool=False):
-        """Convert a JSON Structure schema to an Iceberg schema."""
+    def convert_structure_to_iceberg(self, structure_schema_path: str, structure_record_type: Optional[str], output_path: str, emit_cloudevents_columns: bool=False, output_format: str="arrow"):
+        """Convert a JSON Structure schema to an Iceberg schema.
+        
+        Args:
+            structure_schema_path: Path to the JSON Structure schema file
+            structure_record_type: Record type to convert (or None for the root)
+            output_path: Path to write the Iceberg schema
+            emit_cloudevents_columns: Whether to add CloudEvents columns
+            output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+        """
         schema_file = structure_schema_path
         if not schema_file:
             print("Please specify the JSON Structure schema file")
@@ -114,14 +188,32 @@ class StructureToIcebergConverter:
             ])
 
         iceberg_schema = Schema(*iceberg_fields)
-        arrow_schema = schema_to_pyarrow(iceberg_schema)
-        print(f"Iceberg schema created: {arrow_schema}")
+        print(f"Iceberg schema created: {iceberg_schema}")
 
-        # Write to Iceberg table (for demonstration, using local file system)
-        file_io = PyArrowFileIO()
-        output_file = file_io.new_output("file://"+output_path)
-        with output_file.create(overwrite=True) as f:
-            pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        if output_format == "arrow":
+            # Write as binary PyArrow schema
+            arrow_schema = schema_to_pyarrow(iceberg_schema)
+            file_io = PyArrowFileIO()
+            output_file = file_io.new_output("file://" + output_path)
+            with output_file.create(overwrite=True) as f:
+                pa.output_stream(f).write(arrow_schema.serialize().to_pybytes())
+        else:
+            # Write Iceberg schema as spec-compliant JSON (per Iceberg Table Spec Appendix C)
+            schema_json = {
+                "type": "struct",
+                "schema-id": 0,
+                "fields": [
+                    {
+                        "id": field.field_id,
+                        "name": field.name,
+                        "required": field.required,
+                        "type": iceberg_type_to_json(field.field_type)
+                    }
+                    for field in iceberg_schema.fields
+                ]
+            }
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump(schema_json, f, indent=2)
 
     def resolve_ref(self, ref: str) -> Dict[str, Any]:
         """Resolve a $ref reference."""
@@ -348,8 +440,16 @@ class StructureToIcebergConverter:
         return type_mapping.get(type_name, StringType())
 
 
-def convert_structure_to_iceberg(structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns=False):
-    """Convert a JSON Structure schema to an Iceberg schema."""
+def convert_structure_to_iceberg(structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns=False, output_format="arrow"):
+    """Convert a JSON Structure schema to an Iceberg schema.
+    
+    Args:
+        structure_schema_path: Path to the JSON Structure schema file
+        structure_record_type: Record type to convert (or None for the root)
+        output_path: Path to write the Iceberg schema
+        emit_cloudevents_columns: Whether to add CloudEvents columns
+        output_format: Output format - 'arrow' for binary Arrow IPC (default), 'schema' for JSON
+    """
     converter = StructureToIcebergConverter()
     converter.convert_structure_to_iceberg(
-        structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns)
+        structure_schema_path, structure_record_type, output_path, emit_cloudevents_columns, output_format)

avrotize/validate.py

@@ -0,0 +1,242 @@
+"""Validates JSON instances against Avro or JSON Structure schemas.
+
+This module provides a unified interface for validating JSON data against
+both Avro schemas and JSON Structure schemas.
+"""
+
+import json
+import os
+from typing import Any, Dict, List, Tuple
+
+from avrotize.avrovalidator import AvroValidator, AvroValidationError, validate_json_against_avro
+
+# JSON Structure SDK for validation
+try:
+    from json_structure import SchemaValidator as JStructSchemaValidator
+    from json_structure import InstanceValidator as JStructInstanceValidator
+    HAS_JSTRUCT_SDK = True
+except ImportError:
+    HAS_JSTRUCT_SDK = False
+
+
+class ValidationResult:
+    """Result of validating a JSON instance against a schema."""
+
+    def __init__(self, is_valid: bool, errors: List[str] = None, instance_path: str = None):
+        self.is_valid = is_valid
+        self.errors = errors or []
+        self.instance_path = instance_path
+
+    def __str__(self) -> str:
+        if self.is_valid:
+            return f"✓ Valid" + (f": {self.instance_path}" if self.instance_path else "")
+        else:
+            prefix = f"{self.instance_path}: " if self.instance_path else ""
+            return f"✗ Invalid: {prefix}" + "; ".join(self.errors)
+
+    def __repr__(self) -> str:
+        return f"ValidationResult(is_valid={self.is_valid}, errors={self.errors})"
+
+
+def detect_schema_type(schema: Dict[str, Any]) -> str:
+    """Detects whether a schema is Avro or JSON Structure.
+
+    Args:
+        schema: The parsed schema object
+
+    Returns:
+        'avro' or 'jstruct' or 'unknown'
+    """
+    # JSON Structure schemas have $schema and $id
+    if '$schema' in schema and 'json-structure' in schema.get('$schema', ''):
+        return 'jstruct'
+
+    # Avro schemas have 'type' at root and may have 'namespace', 'fields', etc.
+    if 'type' in schema:
+        schema_type = schema.get('type')
+        # Check for Avro record, enum, array, map, or primitive
+        if schema_type in ('record', 'enum', 'fixed', 'array', 'map'):
+            return 'avro'
+        if schema_type in ('null', 'boolean', 'int', 'long', 'float', 'double', 'bytes', 'string'):
+            return 'avro'
+        # JSON Structure object type
+        if schema_type == 'object' and 'properties' in schema:
+            return 'jstruct'
+
+    # Check if it's a union (list)
+    if isinstance(schema, list):
+        return 'avro'
+
+    return 'unknown'
+
+
+def validate_instance(
+    instance: Any,
+    schema: Dict[str, Any],
+    schema_type: str = None
+) -> ValidationResult:
+    """Validates a JSON instance against a schema.
+
+    Args:
+        instance: The JSON value to validate
+        schema: The schema (Avro or JSON Structure)
+        schema_type: 'avro' or 'jstruct', auto-detected if not provided
+
+    Returns:
+        ValidationResult with validation status and any errors
+    """
+    if schema_type is None:
+        schema_type = detect_schema_type(schema)
+
+    if schema_type == 'avro':
+        errors = validate_json_against_avro(instance, schema)
+        return ValidationResult(is_valid=len(errors) == 0, errors=errors)
+
+    elif schema_type == 'jstruct':
+        if not HAS_JSTRUCT_SDK:
+            return ValidationResult(
+                is_valid=False,
+                errors=["JSON Structure SDK not installed. Install with: pip install json-structure"]
+            )
+        try:
+            validator = JStructInstanceValidator(schema)
+            errors = validator.validate(instance)
+            return ValidationResult(is_valid=len(errors) == 0, errors=errors if errors else [])
+        except Exception as e:
+            return ValidationResult(is_valid=False, errors=[str(e)])
+
+    else:
+        return ValidationResult(
+            is_valid=False,
+            errors=[f"Unknown schema type. Cannot auto-detect schema format."]
+        )
+
+
+def validate_file(
+    instance_file: str,
+    schema_file: str,
+    schema_type: str = None
+) -> List[ValidationResult]:
+    """Validates JSON instance file(s) against a schema file.
+
+    Args:
+        instance_file: Path to JSON file (single object, array, or JSONL)
+        schema_file: Path to schema file (.avsc or .jstruct.json)
+        schema_type: 'avro' or 'jstruct', auto-detected if not provided
+
+    Returns:
+        List of ValidationResult for each instance in the file
+    """
+    # Load schema
+    with open(schema_file, 'r', encoding='utf-8') as f:
+        schema = json.load(f)
+
+    # Auto-detect schema type from file extension if not provided
+    if schema_type is None:
+        if schema_file.endswith('.avsc'):
+            schema_type = 'avro'
+        elif schema_file.endswith('.jstruct.json') or schema_file.endswith('.jstruct'):
+            schema_type = 'jstruct'
+        else:
+            schema_type = detect_schema_type(schema)
+
+    # Load instances
+    with open(instance_file, 'r', encoding='utf-8') as f:
+        content = f.read().strip()
+
+    instances = []
+    instance_paths = []
+
+    # Try as JSON array or object
+    try:
+        data = json.loads(content)
+        if isinstance(data, list):
+            instances = data
+            instance_paths = [f"{instance_file}[{i}]" for i in range(len(data))]
+        else:
+            instances = [data]
+            instance_paths = [instance_file]
+    except json.JSONDecodeError:
+        # Try as JSONL
+        for i, line in enumerate(content.split('\n')):
+            line = line.strip()
+            if line:
+                try:
+                    instances.append(json.loads(line))
+                    instance_paths.append(f"{instance_file}:{i+1}")
+                except json.JSONDecodeError:
+                    pass
+
+    # Validate each instance
+    results = []
+    for instance, path in zip(instances, instance_paths):
+        result = validate_instance(instance, schema, schema_type)
+        result.instance_path = path
+        results.append(result)
+
+    return results
+
+
+def validate_json_instances(
+    input_files: List[str],
+    schema_file: str,
+    schema_type: str = None,
+    verbose: bool = False
+) -> Tuple[int, int]:
+    """Validates multiple JSON instance files against a schema.
+
+    Args:
+        input_files: List of JSON file paths to validate
+        schema_file: Path to schema file
+        schema_type: 'avro' or 'jstruct', auto-detected if not provided
+        verbose: Whether to print validation results
+
+    Returns:
+        Tuple of (valid_count, invalid_count)
+    """
+    valid_count = 0
+    invalid_count = 0
+
+    for input_file in input_files:
+        results = validate_file(input_file, schema_file, schema_type)
+        for result in results:
+            if result.is_valid:
+                valid_count += 1
+                if verbose:
+                    print(result)
+            else:
+                invalid_count += 1
+                if verbose:
+                    print(result)
+
+    return valid_count, invalid_count
+
+
+# Command entry point for avrotize CLI
+def validate(
+    input: List[str],
+    schema: str,
+    schema_type: str = None,
+    quiet: bool = False
+) -> None:
+    """Validates JSON instances against an Avro or JSON Structure schema.
+
+    Args:
+        input: List of JSON files to validate
+        schema: Path to schema file (.avsc or .jstruct.json)
+        schema_type: Schema type ('avro' or 'jstruct'), auto-detected if not provided
+        quiet: Suppress output, exit with code 0 if valid, 1 if invalid
+    """
+    valid_count, invalid_count = validate_json_instances(
+        input_files=input,
+        schema_file=schema,
+        schema_type=schema_type,
+        verbose=not quiet
+    )
+
+    if not quiet:
+        total = valid_count + invalid_count
+        print(f"\nValidation summary: {valid_count}/{total} instances valid")
+
+    if invalid_count > 0:
+        exit(1)

avrotize/xmltoschema.py

@@ -0,0 +1,122 @@
+"""Infers schema from XML files and converts to Avro or JSON Structure format.
+
+This module provides:
+- xml2a: Infer Avro schema from XML files
+- xml2s: Infer JSON Structure schema from XML files
+"""
+
+import json
+import os
+from typing import List
+
+from avrotize.schema_inference import (
+    AvroSchemaInferrer,
+    JsonStructureSchemaInferrer,
+    JsonNode
+)
+
+
+def convert_xml_to_avro(
+    input_files: List[str],
+    avro_schema_file: str,
+    type_name: str = 'Document',
+    avro_namespace: str = '',
+    sample_size: int = 0
+) -> None:
+    """Infers Avro schema from XML files.
+
+    Reads XML files, analyzes their structure, and generates an Avro schema
+    that can represent all the data. Multiple files are analyzed together to
+    produce a unified schema.
+
+    Args:
+        input_files: List of XML file paths to analyze
+        avro_schema_file: Output path for the Avro schema
+        type_name: Name for the root type
+        avro_namespace: Namespace for generated Avro types
+        sample_size: Maximum number of documents to sample (0 = all)
+    """
+    if not input_files:
+        raise ValueError("At least one input file is required")
+
+    xml_strings = _load_xml_strings(input_files, sample_size)
+
+    if not xml_strings:
+        raise ValueError("No valid XML data found in input files")
+
+    inferrer = AvroSchemaInferrer(namespace=avro_namespace)
+    schema = inferrer.infer_from_xml_values(type_name, xml_strings)
+
+    # Ensure output directory exists
+    output_dir = os.path.dirname(avro_schema_file)
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    with open(avro_schema_file, 'w', encoding='utf-8') as f:
+        json.dump(schema, f, indent=2)
+
+
+def convert_xml_to_jstruct(
+    input_files: List[str],
+    jstruct_schema_file: str,
+    type_name: str = 'Document',
+    base_id: str = 'https://example.com/',
+    sample_size: int = 0
+) -> None:
+    """Infers JSON Structure schema from XML files.
+
+    Reads XML files, analyzes their structure, and generates a JSON Structure
+    schema that validates with the official JSON Structure SDK.
+
+    Args:
+        input_files: List of XML file paths to analyze
+        jstruct_schema_file: Output path for the JSON Structure schema
+        type_name: Name for the root type
+        base_id: Base URI for $id generation
+        sample_size: Maximum number of documents to sample (0 = all)
+    """
+    if not input_files:
+        raise ValueError("At least one input file is required")
+
+    xml_strings = _load_xml_strings(input_files, sample_size)
+
+    if not xml_strings:
+        raise ValueError("No valid XML data found in input files")
+
+    inferrer = JsonStructureSchemaInferrer(base_id=base_id)
+    schema = inferrer.infer_from_xml_values(type_name, xml_strings)
+
+    # Ensure output directory exists
+    output_dir = os.path.dirname(jstruct_schema_file)
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    with open(jstruct_schema_file, 'w', encoding='utf-8') as f:
+        json.dump(schema, f, indent=2)
+
+
+def _load_xml_strings(input_files: List[str], sample_size: int) -> List[str]:
+    """Loads XML content from files.
+
+    Each file is treated as a single XML document.
+
+    Args:
+        input_files: List of file paths
+        sample_size: Maximum documents to load (0 = all)
+
+    Returns:
+        List of XML strings
+    """
+    xml_strings: List[str] = []
+
+    for file_path in input_files:
+        if sample_size > 0 and len(xml_strings) >= sample_size:
+            break
+
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read().strip()
+
+        if content:
+            xml_strings.append(content)
+
+    return xml_strings