datacontract-cli 0.10.13__py3-none-any.whl → 0.10.15__py3-none-any.whl

datacontract/imports/bigquery_importer.py

@@ -3,7 +3,7 @@ import logging
 from typing import List
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/imports/dbml_importer.py

@@ -1,11 +1,11 @@
-from pydbml import PyDBML, Database
 from typing import List
 
+from pydbml import Database, PyDBML
 from pyparsing import ParseException
 
 from datacontract.imports.importer import Importer
 from datacontract.imports.sql_importer import map_type_from_sql
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/imports/dbt_importer.py

@@ -1,11 +1,12 @@
 import json
 from typing import TypedDict
 
-from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from dbt.artifacts.resources.v1.components import ColumnInfo
 from dbt.contracts.graph.manifest import Manifest
 
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
+
 
 class DBTImportArgs(TypedDict, total=False):
     """

datacontract/imports/glue_importer.py

@@ -1,11 +1,13 @@
-import boto3
-from typing import List, Dict, Generator
 import re
+from typing import Dict, Generator, List
+
+import boto3
+
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
-    Model,
     Field,
+    Model,
     Server,
 )
 

datacontract/imports/iceberg_importer.py

@@ -0,0 +1,161 @@
+from typing import Any, Dict
+
+from pydantic import ValidationError
+from pyiceberg import types as iceberg_types
+from pyiceberg.schema import Schema
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
+from datacontract.model.exceptions import DataContractException
+
+
+class IcebergImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        schema = load_and_validate_iceberg_schema(source)
+        return import_iceberg(
+            data_contract_specification,
+            schema,
+            import_args.get("iceberg_table"),
+        )
+
+
+def load_and_validate_iceberg_schema(source: str) -> Schema:
+    with open(source, "r") as file:
+        try:
+            return Schema.model_validate_json(file.read())
+        except ValidationError as e:
+            raise DataContractException(
+                type="schema",
+                name="Parse iceberg schema",
+                reason=f"Failed to validate iceberg schema from {source}: {e}",
+                engine="datacontract",
+            )
+
+
+def import_iceberg(
+    data_contract_specification: DataContractSpecification, schema: Schema, table_name: str
+) -> DataContractSpecification:
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    model = Model(type="table", title=table_name)
+
+    for field in schema.fields:
+        model.fields[field.name] = _field_from_nested_field(field)
+
+    data_contract_specification.models[table_name] = model
+    return data_contract_specification
+
+
+def _field_from_nested_field(nested_field: iceberg_types.NestedField) -> Field:
+    """
+    Converts an Iceberg NestedField into a Field object for the data contract.
+
+    Args:
+        nested_field: The Iceberg NestedField to convert.
+
+    Returns:
+        Field: The generated Field object.
+    """
+    field = Field(
+        title=nested_field.name,
+        required=nested_field.required,
+        config=build_field_config(nested_field),
+    )
+
+    if nested_field.doc is not None:
+        field.description = nested_field.doc
+
+    return _type_from_iceberg_type(field, nested_field.field_type)
+
+
+def _type_from_iceberg_type(field: Field, iceberg_type: iceberg_types.IcebergType) -> Field:
+    """
+    Maps Iceberg data types to the Data Contract type system and updates the field.
+
+    Args:
+        field: The Field object to update.
+        iceberg_type: The Iceberg data type to map.
+
+    Returns:
+        Field: The updated Field object.
+    """
+    field.type = _data_type_from_iceberg(iceberg_type)
+
+    if field.type == "array":
+        field.items = _type_from_iceberg_type(Field(required=iceberg_type.element_required), iceberg_type.element_type)
+
+    elif field.type == "map":
+        field.keys = _type_from_iceberg_type(Field(required=True), iceberg_type.key_type)
+        field.values = _type_from_iceberg_type(Field(required=iceberg_type.value_required), iceberg_type.value_type)
+
+    elif field.type == "object":
+        field.fields = {nf.name: _field_from_nested_field(nf) for nf in iceberg_type.fields}
+
+    return field
+
+
+def build_field_config(iceberg_field: iceberg_types.NestedField) -> Dict[str, Any]:
+    config = {}
+
+    if iceberg_field.field_id > 0:
+        config["icebergFieldId"] = iceberg_field.field_id
+
+    if iceberg_field.initial_default is not None:
+        config["icebergInitialDefault"] = iceberg_field.initial_default
+
+    if iceberg_field.write_default is not None:
+        config["icebergWriteDefault"] = iceberg_field.write_default
+
+    return config
+
+
+def _data_type_from_iceberg(type: iceberg_types.IcebergType) -> str:
+    """
+    Convert an Iceberg field type to a datacontract field type
+
+    Args:
+        type: The Iceberg field type
+
+    Returns:
+        str: The datacontract field type
+    """
+    if isinstance(type, iceberg_types.BooleanType):
+        return "boolean"
+    if isinstance(type, iceberg_types.IntegerType):
+        return "integer"
+    if isinstance(type, iceberg_types.LongType):
+        return "long"
+    if isinstance(type, iceberg_types.FloatType):
+        return "float"
+    if isinstance(type, iceberg_types.DoubleType):
+        return "double"
+    if isinstance(type, iceberg_types.DecimalType):
+        return "decimal"
+    if isinstance(type, iceberg_types.DateType):
+        return "date"
+    if isinstance(type, iceberg_types.TimeType):
+        # there isn't a great mapping for the iceberg type "time", just map to string for now
+        return "string"
+    if isinstance(type, iceberg_types.TimestampType):
+        return "timestamp_ntz"
+    if isinstance(type, iceberg_types.TimestamptzType):
+        return "timestamp_tz"
+    if isinstance(type, iceberg_types.StringType):
+        return "string"
+    if isinstance(type, iceberg_types.UUIDType):
+        return "string"
+    if isinstance(type, iceberg_types.BinaryType):
+        return "bytes"
+    if isinstance(type, iceberg_types.FixedType):
+        return "bytes"
+    if isinstance(type, iceberg_types.MapType):
+        return "map"
+    if isinstance(type, iceberg_types.ListType):
+        return "array"
+    if isinstance(type, iceberg_types.StructType):
+        return "object"
+
+    raise ValueError(f"Unknown Iceberg type: {type}")

datacontract/imports/importer.py

@@ -29,6 +29,8 @@ class ImportFormat(str, Enum):
     odcs = "odcs"
     unity = "unity"
     spark = "spark"
+    iceberg = "iceberg"
+    parquet = "parquet"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/imports/importer_factory.py

@@ -1,6 +1,7 @@
 import importlib.util
 import sys
-from datacontract.imports.importer import ImportFormat, Importer
+
+from datacontract.imports.importer import Importer, ImportFormat
 
 
 class ImporterFactory:
@@ -93,3 +94,13 @@ importer_factory.register_lazy_importer(
     module_path="datacontract.imports.dbml_importer",
     class_name="DBMLImporter",
 )
+importer_factory.register_lazy_importer(
+    name=ImportFormat.iceberg,
+    module_path="datacontract.imports.iceberg_importer",
+    class_name="IcebergImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.parquet,
+    module_path="datacontract.imports.parquet_importer",
+    class_name="ParquetImporter",
+)

datacontract/imports/jsonschema_importer.py

@@ -3,7 +3,7 @@ import json
 import fastjsonschema
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Definition
+from datacontract.model.data_contract_specification import DataContractSpecification, Definition, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 
@@ -111,7 +111,8 @@ def schema_to_args(property_schema, is_required: bool = None) -> dict:
     nested_properties = property_schema.get("properties")
     if nested_properties is not None:
         # recursive call for complex nested properties
-        field_kwargs["fields"] = jsonschema_to_args(nested_properties, property_schema["required"])
+        required = property_schema.get("required", [])
+        field_kwargs["fields"] = jsonschema_to_args(nested_properties, required)
 
     return field_kwargs
 

datacontract/imports/odcs_importer.py

@@ -1,47 +1,12 @@
-import datetime
-import logging
-from typing import Any, Dict, List
 import yaml
+
 from datacontract.imports.importer import Importer
+from datacontract.lint.resources import read_resource
 from datacontract.model.data_contract_specification import (
-    Availability,
-    Contact,
     DataContractSpecification,
-    Info,
-    Model,
-    Field,
-    Retention,
-    ServiceLevel,
-    Terms,
 )
 from datacontract.model.exceptions import DataContractException
 
-DATACONTRACT_TYPES = [
-    "string",
-    "text",
-    "varchar",
-    "number",
-    "decimal",
-    "numeric",
-    "int",
-    "integer",
-    "long",
-    "bigint",
-    "float",
-    "double",
-    "boolean",
-    "timestamp",
-    "timestamp_tz",
-    "timestamp_ntz",
-    "date",
-    "array",
-    "bytes",
-    "object",
-    "record",
-    "struct",
-    "null",
-]
-
 
 class OdcsImporter(Importer):
     def import_source(
@@ -52,8 +17,7 @@ class OdcsImporter(Importer):
 
 def import_odcs(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
     try:
-        with open(source, "r") as file:
-            odcs_contract = yaml.safe_load(file.read())
+        odcs_contract = yaml.safe_load(read_resource(source))
 
     except Exception as e:
         raise DataContractException(
@@ -64,137 +28,30 @@ def import_odcs(data_contract_specification: DataContractSpecification, source:
             original_exception=e,
         )
 
-    data_contract_specification.id = odcs_contract["uuid"]
-    data_contract_specification.info = import_info(odcs_contract)
-    data_contract_specification.terms = import_terms(odcs_contract)
-    data_contract_specification.servicelevels = import_servicelevels(odcs_contract)
-    data_contract_specification.models = import_models(odcs_contract)
-
-    return data_contract_specification
-
-
-def import_info(odcs_contract: Dict[str, Any]) -> Info:
-    info = Info(title=odcs_contract.get("quantumName"), version=odcs_contract.get("version"))
-
-    if odcs_contract.get("description").get("purpose") is not None:
-        info.description = odcs_contract.get("description").get("purpose")
-
-    if odcs_contract.get("datasetDomain") is not None:
-        info.owner = odcs_contract.get("datasetDomain")
-
-    if odcs_contract.get("productDl") is not None or odcs_contract.get("productFeedbackUrl") is not None:
-        contact = Contact()
-        if odcs_contract.get("productDl") is not None:
-            contact.name = odcs_contract.get("productDl")
-        if odcs_contract.get("productFeedbackUrl") is not None:
-            contact.url = odcs_contract.get("productFeedbackUrl")
-
-        info.contact = contact
-
-    return info
+    odcs_kind = odcs_contract.get("kind")
+    odcs_api_version = odcs_contract.get("apiVersion")
 
+    # if odcs_kind is not DataContract throw exception
+    if odcs_kind != "DataContract":
+        raise DataContractException(
+            type="schema",
+            name="Importing ODCS contract",
+            reason=f"Unsupported ODCS kind: {odcs_kind}. Is this a valid ODCS data contract?",
+            engine="datacontract",
+        )
 
-def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None:
-    if (
-        odcs_contract.get("description").get("usage") is not None
-        or odcs_contract.get("description").get("limitations") is not None
-        or odcs_contract.get("price") is not None
-    ):
-        terms = Terms()
-        if odcs_contract.get("description").get("usage") is not None:
-            terms.usage = odcs_contract.get("description").get("usage")
-        if odcs_contract.get("description").get("limitations") is not None:
-            terms.limitations = odcs_contract.get("description").get("limitations")
-        if odcs_contract.get("price") is not None:
-            terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}"
-
-        return terms
-    else:
-        return None
-
-
-def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel:
-    # find the two properties we can map (based on the examples)
-    sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else []
-    availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None)
-    retention = next((p for p in sla_properties if p["property"] == "retention"), None)
-
-    if availability is not None or retention is not None:
-        servicelevel = ServiceLevel()
-
-        if availability is not None:
-            value = availability.get("value")
-            if isinstance(value, datetime.datetime):
-                value = value.isoformat()
-            servicelevel.availability = Availability(description=value)
-
-        if retention is not None:
-            servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}")
-
-        return servicelevel
-    else:
-        return None
-
-
-def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
-    custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties"))
-
-    odcs_tables = odcs_contract.get("dataset") if odcs_contract.get("dataset") is not None else []
-    result = {}
-
-    for table in odcs_tables:
-        description = table.get("description") if table.get("description") is not None else ""
-        model = Model(description=" ".join(description.splitlines()), type="table")
-        model.fields = import_fields(table.get("columns"), custom_type_mappings)
-        result[table.get("table")] = model
-
-    return result
-
-
-def import_fields(odcs_columns: Dict[str, Any], custom_type_mappings: Dict[str, str]) -> Dict[str, Field]:
-    logger = logging.getLogger(__name__)
-    result = {}
-
-    for column in odcs_columns:
-        mapped_type = map_type(column.get("logicalType"), custom_type_mappings)
-        if mapped_type is not None:
-            description = column.get("description") if column.get("description") is not None else ""
-            field = Field(
-                description=" ".join(description.splitlines()),
-                type=mapped_type,
-                title=column.get("businessName") if column.get("businessName") is not None else "",
-                required=not column.get("isNullable") if column.get("isNullable") is not None else False,
-                primary=column.get("isPrimary") if column.get("isPrimary") is not None else False,
-                unique=column.get("isUnique") if column.get("isUnique") is not None else False,
-                classification=column.get("classification") if column.get("classification") is not None else "",
-                tags=column.get("tags") if column.get("tags") is not None else [],
-            )
-            result[column["column"]] = field
-        else:
-            logger.info(
-                f"Can't properly map {column.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{column.get('logicalName')}' that defines your expected type as the 'value'"
-            )
-
-    return result
+    if odcs_api_version.startswith("v2."):
+        from datacontract.imports.odcs_v2_importer import import_odcs_v2
 
+        return import_odcs_v2(data_contract_specification, source)
+    elif odcs_api_version.startswith("v3."):
+        from datacontract.imports.odcs_v3_importer import import_odcs_v3
 
-def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
-    t = odcs_type.lower()
-    if t in DATACONTRACT_TYPES:
-        return t
-    elif custom_mappings.get(t) is not None:
-        return custom_mappings.get(t)
+        return import_odcs_v3(data_contract_specification, source)
     else:
-        return None
-
-
-def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]:
-    result = {}
-    if odcs_custom_properties is not None:
-        for prop in odcs_custom_properties:
-            if prop["property"].startswith("dc_mapping_"):
-                odcs_type_name = prop["property"].substring(11)
-                datacontract_type = prop["value"]
-                result[odcs_type_name] = datacontract_type
-
-    return result
+        raise DataContractException(
+            type="schema",
+            name="Importing ODCS contract",
+            reason=f"Unsupported ODCS API version: {odcs_api_version}",
+            engine="datacontract",
+        )

datacontract/imports/odcs_v2_importer.py

@@ -0,0 +1,177 @@
+import datetime
+import logging
+from typing import Any, Dict, List
+
+import yaml
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import (
+    DATACONTRACT_TYPES,
+    Availability,
+    Contact,
+    DataContractSpecification,
+    Field,
+    Info,
+    Model,
+    Retention,
+    ServiceLevel,
+    Terms,
+)
+from datacontract.model.exceptions import DataContractException
+
+
+class OdcsImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_odcs_v2(data_contract_specification, source)
+
+
+def import_odcs_v2(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    try:
+        with open(source, "r") as file:
+            odcs_contract = yaml.safe_load(file.read())
+
+    except Exception as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse ODCS contract",
+            reason=f"Failed to parse odcs contract from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    data_contract_specification.id = odcs_contract["uuid"]
+    data_contract_specification.info = import_info(odcs_contract)
+    data_contract_specification.terms = import_terms(odcs_contract)
+    data_contract_specification.servicelevels = import_servicelevels(odcs_contract)
+    data_contract_specification.models = import_models(odcs_contract)
+
+    return data_contract_specification
+
+
+def import_info(odcs_contract: Dict[str, Any]) -> Info:
+    info = Info(title=odcs_contract.get("quantumName"), version=odcs_contract.get("version"))
+
+    if odcs_contract.get("description").get("purpose") is not None:
+        info.description = odcs_contract.get("description").get("purpose")
+
+    if odcs_contract.get("datasetDomain") is not None:
+        info.owner = odcs_contract.get("datasetDomain")
+
+    if odcs_contract.get("productDl") is not None or odcs_contract.get("productFeedbackUrl") is not None:
+        contact = Contact()
+        if odcs_contract.get("productDl") is not None:
+            contact.name = odcs_contract.get("productDl")
+        if odcs_contract.get("productFeedbackUrl") is not None:
+            contact.url = odcs_contract.get("productFeedbackUrl")
+
+        info.contact = contact
+
+    return info
+
+
+def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None:
+    if (
+        odcs_contract.get("description").get("usage") is not None
+        or odcs_contract.get("description").get("limitations") is not None
+        or odcs_contract.get("price") is not None
+    ):
+        terms = Terms()
+        if odcs_contract.get("description").get("usage") is not None:
+            terms.usage = odcs_contract.get("description").get("usage")
+        if odcs_contract.get("description").get("limitations") is not None:
+            terms.limitations = odcs_contract.get("description").get("limitations")
+        if odcs_contract.get("price") is not None:
+            terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}"
+
+        return terms
+    else:
+        return None
+
+
+def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel:
+    # find the two properties we can map (based on the examples)
+    sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else []
+    availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None)
+    retention = next((p for p in sla_properties if p["property"] == "retention"), None)
+
+    if availability is not None or retention is not None:
+        servicelevel = ServiceLevel()
+
+        if availability is not None:
+            value = availability.get("value")
+            if isinstance(value, datetime.datetime):
+                value = value.isoformat()
+            servicelevel.availability = Availability(description=value)
+
+        if retention is not None:
+            servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}")
+
+        return servicelevel
+    else:
+        return None
+
+
+def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
+    custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties"))
+
+    odcs_tables = odcs_contract.get("dataset") if odcs_contract.get("dataset") is not None else []
+    result = {}
+
+    for table in odcs_tables:
+        description = table.get("description") if table.get("description") is not None else ""
+        model = Model(description=" ".join(description.splitlines()), type="table")
+        model.fields = import_fields(table.get("columns"), custom_type_mappings)
+        result[table.get("table")] = model
+
+    return result
+
+
+def import_fields(odcs_columns: Dict[str, Any], custom_type_mappings: Dict[str, str]) -> Dict[str, Field]:
+    logger = logging.getLogger(__name__)
+    result = {}
+
+    for column in odcs_columns:
+        mapped_type = map_type(column.get("logicalType"), custom_type_mappings)
+        if mapped_type is not None:
+            description = column.get("description") if column.get("description") is not None else ""
+            field = Field(
+                description=" ".join(description.splitlines()),
+                type=mapped_type,
+                title=column.get("businessName") if column.get("businessName") is not None else "",
+                required=not column.get("isNullable") if column.get("isNullable") is not None else False,
+                primary=column.get("isPrimary") if column.get("isPrimary") is not None else False,
+                unique=column.get("isUnique") if column.get("isUnique") is not None else False,
+                classification=column.get("classification") if column.get("classification") is not None else "",
+                tags=column.get("tags") if column.get("tags") is not None else [],
+            )
+            result[column["column"]] = field
+        else:
+            logger.info(
+                f"Can't properly map {column.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{column.get('logicalName')}' that defines your expected type as the 'value'"
+            )
+
+    return result
+
+
+def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
+    t = odcs_type.lower()
+    if t in DATACONTRACT_TYPES:
+        return t
+    elif custom_mappings.get(t) is not None:
+        return custom_mappings.get(t)
+    else:
+        return None
+
+
+def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]:
+    result = {}
+    if odcs_custom_properties is not None:
+        for prop in odcs_custom_properties:
+            if prop["property"].startswith("dc_mapping_"):
+                odcs_type_name = prop["property"].substring(11)
+                datacontract_type = prop["value"]
+                result[odcs_type_name] = datacontract_type
+
+    return result