datacontract-cli 0.10.23__py3-none-any.whl → 0.10.25__py3-none-any.whl

datacontract/imports/importer.py

@@ -1,7 +1,8 @@
 from abc import ABC, abstractmethod
 from enum import Enum
 
-from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract_specification.model import DataContractSpecification
+from open_data_contract_standard.model import OpenDataContractStandard
 
 
 class Importer(ABC):
@@ -14,7 +15,7 @@ class Importer(ABC):
         data_contract_specification: DataContractSpecification,
         source: str,
         import_args: dict,
-    ) -> DataContractSpecification:
+    ) -> DataContractSpecification | OpenDataContractStandard:
         pass
 
 
@@ -32,6 +33,8 @@ class ImportFormat(str, Enum):
     iceberg = "iceberg"
     parquet = "parquet"
     csv = "csv"
+    protobuf = "protobuf"
+    excel = "excel"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/imports/importer_factory.py

@@ -109,3 +109,13 @@ importer_factory.register_lazy_importer(
     module_path="datacontract.imports.csv_importer",
     class_name="CsvImporter",
 )
+importer_factory.register_lazy_importer(
+    name=ImportFormat.protobuf,
+    module_path="datacontract.imports.protobuf_importer",
+    class_name="ProtoBufImporter",
+)
+importer_factory.register_lazy_importer(
+    name=ImportFormat.excel,
+    module_path="datacontract.imports.excel_importer",
+    class_name="ExcelImporter",
+)

datacontract/imports/odcs_v3_importer.py

@@ -1,9 +1,11 @@
 import datetime
 import logging
+import re
 from typing import Any, Dict, List
 from venv import logger
 
-import yaml
+from datacontract_specification.model import Quality
+from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard, SchemaProperty
 
 from datacontract.imports.importer import Importer
 from datacontract.lint.resources import read_resource
@@ -14,9 +16,9 @@ from datacontract.model.data_contract_specification import (
     Field,
     Info,
     Model,
-    Quality,
     Retention,
     Server,
+    ServerRole,
     ServiceLevel,
     Terms,
 )
@@ -39,7 +41,7 @@ def import_odcs_v3_from_str(
     data_contract_specification: DataContractSpecification, source_str: str
 ) -> DataContractSpecification:
     try:
-        odcs_contract = yaml.safe_load(source_str)
+        odcs = OpenDataContractStandard.from_string(source_str)
     except Exception as e:
         raise DataContractException(
             type="schema",
@@ -49,129 +51,140 @@ def import_odcs_v3_from_str(
             original_exception=e,
         )
 
-    data_contract_specification.id = odcs_contract["id"]
-    data_contract_specification.info = import_info(odcs_contract)
-    data_contract_specification.servers = import_servers(odcs_contract)
-    data_contract_specification.terms = import_terms(odcs_contract)
-    data_contract_specification.servicelevels = import_servicelevels(odcs_contract)
-    data_contract_specification.models = import_models(odcs_contract)
-    data_contract_specification.tags = import_tags(odcs_contract)
+    return import_from_odcs_model(data_contract_specification, odcs)
 
+
+def import_from_odcs_model(data_contract_specification, odcs):
+    data_contract_specification.id = odcs.id
+    data_contract_specification.info = import_info(odcs)
+    data_contract_specification.servers = import_servers(odcs)
+    data_contract_specification.terms = import_terms(odcs)
+    data_contract_specification.servicelevels = import_servicelevels(odcs)
+    data_contract_specification.models = import_models(odcs)
+    data_contract_specification.tags = import_tags(odcs)
     return data_contract_specification
 
 
-def import_info(odcs_contract: Dict[str, Any]) -> Info:
+def import_info(odcs: Any) -> Info:
     info = Info()
 
-    info.title = odcs_contract.get("name") if odcs_contract.get("name") is not None else ""
+    info.title = odcs.name if odcs.name is not None else ""
 
-    if odcs_contract.get("version") is not None:
-        info.version = odcs_contract.get("version")
+    if odcs.version is not None:
+        info.version = odcs.version
 
     # odcs.description.purpose => datacontract.description
-    if odcs_contract.get("description") is not None and odcs_contract.get("description").get("purpose") is not None:
-        info.description = odcs_contract.get("description").get("purpose")
+    if odcs.description is not None and odcs.description.purpose is not None:
+        info.description = odcs.description.purpose
 
     # odcs.domain => datacontract.owner
-    if odcs_contract.get("domain") is not None:
-        info.owner = odcs_contract.get("domain")
+    owner = get_owner(odcs.customProperties)
+    if owner is not None:
+        info.owner = owner
 
     # add dataProduct as custom property
-    if odcs_contract.get("dataProduct") is not None:
-        info.dataProduct = odcs_contract.get("dataProduct")
+    if odcs.dataProduct is not None:
+        info.dataProduct = odcs.dataProduct
 
     # add tenant as custom property
-    if odcs_contract.get("tenant") is not None:
-        info.tenant = odcs_contract.get("tenant")
+    if odcs.tenant is not None:
+        info.tenant = odcs.tenant
 
     return info
 
 
-def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None:
-    if odcs_contract.get("servers") is None:
+def import_server_roles(roles: List[Dict]) -> List[ServerRole] | None:
+    if roles is None:
+        return None
+    result = []
+    for role in roles:
+        server_role = ServerRole()
+        server_role.name = role.role
+        server_role.description = role.description
+        result.append(server_role)
+
+
+def import_servers(odcs: OpenDataContractStandard) -> Dict[str, Server] | None:
+    if odcs.servers is None:
         return None
     servers = {}
-    for odcs_server in odcs_contract.get("servers"):
-        server_name = odcs_server.get("server")
+    for odcs_server in odcs.servers:
+        server_name = odcs_server.server
         if server_name is None:
             logger.warning("Server name is missing, skipping server")
             continue
 
         server = Server()
-        server.type = odcs_server.get("type")
-        server.description = odcs_server.get("description")
-        server.environment = odcs_server.get("environment")
-        server.format = odcs_server.get("format")
-        server.project = odcs_server.get("project")
-        server.dataset = odcs_server.get("dataset")
-        server.path = odcs_server.get("path")
-        server.delimiter = odcs_server.get("delimiter")
-        server.endpointUrl = odcs_server.get("endpointUrl")
-        server.location = odcs_server.get("location")
-        server.account = odcs_server.get("account")
-        server.database = odcs_server.get("database")
-        server.schema_ = odcs_server.get("schema")
-        server.host = odcs_server.get("host")
-        server.port = odcs_server.get("port")
-        server.catalog = odcs_server.get("catalog")
-        server.topic = odcs_server.get("topic")
-        server.http_path = odcs_server.get("http_path")
-        server.token = odcs_server.get("token")
-        server.dataProductId = odcs_server.get("dataProductId")
-        server.outputPortId = odcs_server.get("outputPortId")
-        server.driver = odcs_server.get("driver")
-        server.roles = odcs_server.get("roles")
-
+        server.type = odcs_server.type
+        server.description = odcs_server.description
+        server.environment = odcs_server.environment
+        server.format = odcs_server.format
+        server.project = odcs_server.project
+        server.dataset = odcs_server.dataset
+        server.path = odcs_server.path
+        server.delimiter = odcs_server.delimiter
+        server.endpointUrl = odcs_server.endpointUrl
+        server.location = odcs_server.location
+        server.account = odcs_server.account
+        server.database = odcs_server.database
+        server.schema_ = odcs_server.schema_
+        server.host = odcs_server.host
+        server.port = odcs_server.port
+        server.catalog = odcs_server.catalog
+        server.topic = getattr(odcs_server, "topic", None)
+        server.http_path = getattr(odcs_server, "http_path", None)
+        server.token = getattr(odcs_server, "token", None)
+        server.driver = getattr(odcs_server, "driver", None)
+        server.roles = import_server_roles(odcs_server.roles)
+        server.storageAccount = (
+            re.search(r"(?:@|://)([^.]+)\.", odcs_server.location, re.IGNORECASE) if server.type == "azure" else None
+        )
         servers[server_name] = server
     return servers
 
 
-def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None:
-    if odcs_contract.get("description") is None:
+def import_terms(odcs: Any) -> Terms | None:
+    if odcs.description is None:
         return None
-    if (
-        odcs_contract.get("description").get("usage") is not None
-        or odcs_contract.get("description").get("limitations") is not None
-        or odcs_contract.get("price") is not None
-    ):
+    if odcs.description.usage is not None or odcs.description.limitations is not None or odcs.price is not None:
         terms = Terms()
-        if odcs_contract.get("description").get("usage") is not None:
-            terms.usage = odcs_contract.get("description").get("usage")
-        if odcs_contract.get("description").get("limitations") is not None:
-            terms.limitations = odcs_contract.get("description").get("limitations")
-        if odcs_contract.get("price") is not None:
-            terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}"
+        if odcs.description.usage is not None:
+            terms.usage = odcs.description.usage
+        if odcs.description.limitations is not None:
+            terms.limitations = odcs.description.limitations
+        if odcs.price is not None:
+            terms.billing = f"{odcs.price.priceAmount} {odcs.price.priceCurrency} / {odcs.price.priceUnit}"
 
         return terms
     else:
         return None
 
 
-def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel:
+def import_servicelevels(odcs: Any) -> ServiceLevel:
     # find the two properties we can map (based on the examples)
-    sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else []
-    availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None)
-    retention = next((p for p in sla_properties if p["property"] == "retention"), None)
+    sla_properties = odcs.slaProperties if odcs.slaProperties is not None else []
+    availability = next((p for p in sla_properties if p.property == "generalAvailability"), None)
+    retention = next((p for p in sla_properties if p.property == "retention"), None)
 
     if availability is not None or retention is not None:
         servicelevel = ServiceLevel()
 
         if availability is not None:
-            value = availability.get("value")
+            value = availability.value
             if isinstance(value, datetime.datetime):
                 value = value.isoformat()
             servicelevel.availability = Availability(description=value)
 
         if retention is not None:
-            servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}")
+            servicelevel.retention = Retention(period=f"{retention.value}{retention.unit}")
 
         return servicelevel
     else:
         return None
 
 
-def get_server_type(odcs_contract: Dict[str, Any]) -> str | None:
-    servers = import_servers(odcs_contract)
+def get_server_type(odcs: OpenDataContractStandard) -> str | None:
+    servers = import_servers(odcs)
     if servers is None or len(servers) == 0:
         return None
     # get first server from map
@@ -179,49 +192,106 @@ def get_server_type(odcs_contract: Dict[str, Any]) -> str | None:
     return server.type
 
 
-def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
-    custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties"))
+def import_models(odcs: Any) -> Dict[str, Model]:
+    custom_type_mappings = get_custom_type_mappings(odcs.customProperties)
 
-    odcs_schemas = odcs_contract.get("schema") if odcs_contract.get("schema") is not None else []
+    odcs_schemas = odcs.schema_ if odcs.schema_ is not None else []
     result = {}
 
     for odcs_schema in odcs_schemas:
-        schema_name = odcs_schema.get("name")
-        schema_physical_name = odcs_schema.get("physicalName")
-        schema_description = odcs_schema.get("description") if odcs_schema.get("description") is not None else ""
+        schema_name = odcs_schema.name
+        schema_physical_name = odcs_schema.physicalName
+        schema_description = odcs_schema.description if odcs_schema.description is not None else ""
         model_name = schema_physical_name if schema_physical_name is not None else schema_name
-        model = Model(description=" ".join(schema_description.splitlines()), type="table")
-        model.fields = import_fields(
-            odcs_schema.get("properties"), custom_type_mappings, server_type=get_server_type(odcs_contract)
-        )
-        if odcs_schema.get("quality") is not None:
-            # convert dict to pydantic model
-
-            model.quality = [Quality.model_validate(q) for q in odcs_schema.get("quality")]
+        model = Model(description=" ".join(schema_description.splitlines()) if schema_description else "", type="table")
+        model.fields = import_fields(odcs_schema.properties, custom_type_mappings, server_type=get_server_type(odcs))
+        if odcs_schema.quality is not None:
+            model.quality = convert_quality_list(odcs_schema.quality)
         model.title = schema_name
-        if odcs_schema.get("dataGranularityDescription") is not None:
-            model.config = {"dataGranularityDescription": odcs_schema.get("dataGranularityDescription")}
+        if odcs_schema.dataGranularityDescription is not None:
+            model.config = {"dataGranularityDescription": odcs_schema.dataGranularityDescription}
         result[model_name] = model
 
     return result
 
 
-def import_field_config(odcs_property: Dict[str, Any], server_type=None) -> Dict[str, Any]:
+def convert_quality_list(odcs_quality_list):
+    """Convert a list of ODCS DataQuality objects to datacontract Quality objects"""
+    quality_list = []
+
+    if odcs_quality_list is not None:
+        for odcs_quality in odcs_quality_list:
+            quality = Quality(type=odcs_quality.type)
+
+            if odcs_quality.description is not None:
+                quality.description = odcs_quality.description
+            if odcs_quality.query is not None:
+                quality.query = odcs_quality.query
+            if odcs_quality.mustBe is not None:
+                quality.mustBe = odcs_quality.mustBe
+            if odcs_quality.mustNotBe is not None:
+                quality.mustNotBe = odcs_quality.mustNotBe
+            if odcs_quality.mustBeGreaterThan is not None:
+                quality.mustBeGreaterThan = odcs_quality.mustBeGreaterThan
+            if odcs_quality.mustBeGreaterOrEqualTo is not None:
+                quality.mustBeGreaterThanOrEqualTo = odcs_quality.mustBeGreaterOrEqualTo
+            if odcs_quality.mustBeLessThan is not None:
+                quality.mustBeLessThan = odcs_quality.mustBeLessThan
+            if odcs_quality.mustBeLessOrEqualTo is not None:
+                quality.mustBeLessThanOrEqualTo = odcs_quality.mustBeLessOrEqualTo
+            if odcs_quality.mustBeBetween is not None:
+                quality.mustBeBetween = odcs_quality.mustBeBetween
+            if odcs_quality.mustNotBeBetween is not None:
+                quality.mustNotBeBetween = odcs_quality.mustNotBeBetween
+            if odcs_quality.engine is not None:
+                quality.engine = odcs_quality.engine
+            if odcs_quality.implementation is not None:
+                quality.implementation = odcs_quality.implementation
+            if odcs_quality.businessImpact is not None:
+                quality.model_extra["businessImpact"] = odcs_quality.businessImpact
+            if odcs_quality.dimension is not None:
+                quality.model_extra["dimension"] = odcs_quality.dimension
+            if odcs_quality.rule is not None:
+                quality.model_extra["rule"] = odcs_quality.rule
+            if odcs_quality.schedule is not None:
+                quality.model_extra["schedule"] = odcs_quality.schedule
+            if odcs_quality.scheduler is not None:
+                quality.model_extra["scheduler"] = odcs_quality.scheduler
+            if odcs_quality.severity is not None:
+                quality.model_extra["severity"] = odcs_quality.severity
+            if odcs_quality.method is not None:
+                quality.model_extra["method"] = odcs_quality.method
+            if odcs_quality.customProperties is not None:
+                quality.model_extra["customProperties"] = []
+                for item in odcs_quality.customProperties:
+                    quality.model_extra["customProperties"].append(
+                        {
+                            "property": item.property,
+                            "value": item.value,
+                        }
+                    )
+
+            quality_list.append(quality)
+
+    return quality_list
+
+
+def import_field_config(odcs_property: SchemaProperty, server_type=None) -> Dict[str, Any]:
     config = {}
-    if odcs_property.get("criticalDataElement") is not None:
-        config["criticalDataElement"] = odcs_property.get("criticalDataElement")
-    if odcs_property.get("encryptedName") is not None:
-        config["encryptedName"] = odcs_property.get("encryptedName")
-    if odcs_property.get("partitionKeyPosition") is not None:
-        config["partitionKeyPosition"] = odcs_property.get("partitionKeyPosition")
-    if odcs_property.get("partitioned") is not None:
-        config["partitioned"] = odcs_property.get("partitioned")
-
-    if odcs_property.get("customProperties") is not None and isinstance(odcs_property.get("customProperties"), list):
-        for item in odcs_property.get("customProperties"):
-            config[item["property"]] = item["value"]
-
-    physical_type = odcs_property.get("physicalType")
+    if odcs_property.criticalDataElement is not None:
+        config["criticalDataElement"] = odcs_property.criticalDataElement
+    if odcs_property.encryptedName is not None:
+        config["encryptedName"] = odcs_property.encryptedName
+    if odcs_property.partitionKeyPosition is not None:
+        config["partitionKeyPosition"] = odcs_property.partitionKeyPosition
+    if odcs_property.partitioned is not None:
+        config["partitioned"] = odcs_property.partitioned
+
+    if odcs_property.customProperties is not None:
+        for item in odcs_property.customProperties:
+            config[item.property] = item.value
+
+    physical_type = odcs_property.physicalType
     if physical_type is not None:
         if server_type == "postgres" or server_type == "postgresql":
             config["postgresType"] = physical_type
@@ -241,13 +311,13 @@ def import_field_config(odcs_property: Dict[str, Any], server_type=None) -> Dict
     return config
 
 
-def has_composite_primary_key(odcs_properties) -> bool:
-    primary_keys = [prop for prop in odcs_properties if prop.get("primaryKey") is not None and prop.get("primaryKey")]
+def has_composite_primary_key(odcs_properties: List[SchemaProperty]) -> bool:
+    primary_keys = [prop for prop in odcs_properties if prop.primaryKey is not None and prop.primaryKey]
     return len(primary_keys) > 1
 
 
 def import_fields(
-    odcs_properties: Dict[str, Any], custom_type_mappings: Dict[str, str], server_type
+    odcs_properties: List[SchemaProperty], custom_type_mappings: Dict[str, str], server_type
 ) -> Dict[str, Field]:
     logger = logging.getLogger(__name__)
     result = {}
@@ -256,31 +326,51 @@ def import_fields(
         return result
 
     for odcs_property in odcs_properties:
-        mapped_type = map_type(odcs_property.get("logicalType"), custom_type_mappings)
+        mapped_type = map_type(odcs_property.logicalType, custom_type_mappings)
         if mapped_type is not None:
-            property_name = odcs_property["name"]
-            description = odcs_property.get("description") if odcs_property.get("description") is not None else None
+            property_name = odcs_property.name
+            description = odcs_property.description if odcs_property.description is not None else None
             field = Field(
                 description=" ".join(description.splitlines()) if description is not None else None,
                 type=mapped_type,
-                title=odcs_property.get("businessName"),
-                required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False,
-                primaryKey=odcs_property.get("primaryKey")
-                if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None
+                title=odcs_property.businessName,
+                required=odcs_property.required if odcs_property.required is not None else None,
+                primaryKey=odcs_property.primaryKey
+                if not has_composite_primary_key(odcs_properties) and odcs_property.primaryKey is not None
                 else False,
-                unique=odcs_property.get("unique"),
-                examples=odcs_property.get("examples") if odcs_property.get("examples") is not None else None,
-                classification=odcs_property.get("classification")
-                if odcs_property.get("classification") is not None
-                else "",
-                tags=odcs_property.get("tags") if odcs_property.get("tags") is not None else None,
-                quality=odcs_property.get("quality") if odcs_property.get("quality") is not None else [],
+                unique=odcs_property.unique if odcs_property.unique else None,
+                examples=odcs_property.examples if odcs_property.examples is not None else None,
+                classification=odcs_property.classification if odcs_property.classification is not None else None,
+                tags=odcs_property.tags if odcs_property.tags is not None else None,
+                quality=convert_quality_list(odcs_property.quality),
+                fields=import_fields(odcs_property.properties, custom_type_mappings, server_type)
+                if odcs_property.properties is not None
+                else {},
                 config=import_field_config(odcs_property, server_type),
+                format=getattr(odcs_property, "format", None),
             )
+            # mapped_type is array
+            if field.type == "array" and odcs_property.items is not None:
+                # nested array object
+                if odcs_property.items.logicalType == "object":
+                    field.items = Field(
+                        type="object",
+                        fields=import_fields(odcs_property.items.properties, custom_type_mappings, server_type),
+                    )
+                # array of simple type
+                elif odcs_property.items.logicalType is not None:
+                    field.items = Field(type=odcs_property.items.logicalType)
+
+            # enum from quality validValues as enum
+            if field.type == "string":
+                for q in field.quality:
+                    if hasattr(q, "validValues"):
+                        field.enum = q.validValues
+
             result[property_name] = field
         else:
             logger.info(
-                f"Can't map {odcs_property.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{odcs_property.get('logicalName')}' that defines your expected type as the 'value'"
+                f"Can't map {odcs_property.name} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{odcs_property.logicalType}' that defines your expected type as the 'value'"
             )
 
     return result
@@ -298,19 +388,28 @@ def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
         return None
 
 
-def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]:
+def get_custom_type_mappings(odcs_custom_properties: List[CustomProperty]) -> Dict[str, str]:
     result = {}
     if odcs_custom_properties is not None:
         for prop in odcs_custom_properties:
-            if prop["property"].startswith("dc_mapping_"):
-                odcs_type_name = prop["property"].substring(11)
-                datacontract_type = prop["value"]
+            if prop.property.startswith("dc_mapping_"):
+                odcs_type_name = prop.property[11:]  # Changed substring to slice
+                datacontract_type = prop.value
                 result[odcs_type_name] = datacontract_type
 
     return result
 
 
-def import_tags(odcs_contract) -> List[str] | None:
-    if odcs_contract.get("tags") is None:
+def get_owner(odcs_custom_properties: List[CustomProperty]) -> str | None:
+    if odcs_custom_properties is not None:
+        for prop in odcs_custom_properties:
+            if prop.property == "owner":
+                return prop.value
+
+    return None
+
+
+def import_tags(odcs: OpenDataContractStandard) -> List[str] | None:
+    if odcs.tags is None:
         return None
-    return odcs_contract.get("tags")
+    return odcs.tags