datacontract-cli 0.10.13__py3-none-any.whl → 0.10.15__py3-none-any.whl

datacontract/imports/odcs_v3_importer.py

@@ -0,0 +1,309 @@
+import datetime
+import logging
+from typing import Any, Dict, List
+from venv import logger
+
+import yaml
+
+from datacontract.imports.importer import Importer
+from datacontract.lint.resources import read_resource
+from datacontract.model.data_contract_specification import (
+    DATACONTRACT_TYPES,
+    Availability,
+    DataContractSpecification,
+    Field,
+    Info,
+    Model,
+    Retention,
+    Server,
+    ServiceLevel,
+    Terms,
+)
+from datacontract.model.exceptions import DataContractException
+
+
+class OdcsImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_odcs_v3(data_contract_specification, source)
+
+
+def import_odcs_v3(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    source_str = read_resource(source)
+    return import_odcs_v3_from_str(data_contract_specification, source_str)
+
+
+def import_odcs_v3_from_str(
+    data_contract_specification: DataContractSpecification, source_str: str
+) -> DataContractSpecification:
+    try:
+        odcs_contract = yaml.safe_load(source_str)
+    except Exception as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse ODCS contract",
+            reason=f"Failed to parse odcs contract from {source_str}",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    data_contract_specification.id = odcs_contract["id"]
+    data_contract_specification.info = import_info(odcs_contract)
+    data_contract_specification.servers = import_servers(odcs_contract)
+    data_contract_specification.terms = import_terms(odcs_contract)
+    data_contract_specification.servicelevels = import_servicelevels(odcs_contract)
+    data_contract_specification.models = import_models(odcs_contract)
+    data_contract_specification.tags = import_tags(odcs_contract)
+
+    return data_contract_specification
+
+
+def import_info(odcs_contract: Dict[str, Any]) -> Info:
+    info = Info()
+
+    info.title = odcs_contract.get("name") if odcs_contract.get("name") is not None else ""
+
+    if odcs_contract.get("version") is not None:
+        info.version = odcs_contract.get("version")
+
+    # odcs.description.purpose => datacontract.description
+    if odcs_contract.get("description") is not None and odcs_contract.get("description").get("purpose") is not None:
+        info.description = odcs_contract.get("description").get("purpose")
+
+    # odcs.domain => datacontract.owner
+    if odcs_contract.get("domain") is not None:
+        info.owner = odcs_contract.get("domain")
+
+    # add dataProduct as custom property
+    if odcs_contract.get("dataProduct") is not None:
+        info.dataProduct = odcs_contract.get("dataProduct")
+
+    # add tenant as custom property
+    if odcs_contract.get("tenant") is not None:
+        info.tenant = odcs_contract.get("tenant")
+
+    return info
+
+
+def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None:
+    if odcs_contract.get("servers") is None:
+        return None
+    servers = {}
+    for odcs_server in odcs_contract.get("servers"):
+        server_name = odcs_server.get("server")
+        if server_name is None:
+            logger.warning("Server name is missing, skipping server")
+            continue
+
+        server = Server()
+        server.type = odcs_server.get("type")
+        server.description = odcs_server.get("description")
+        server.environment = odcs_server.get("environment")
+        server.format = odcs_server.get("format")
+        server.project = odcs_server.get("project")
+        server.dataset = odcs_server.get("dataset")
+        server.path = odcs_server.get("path")
+        server.delimiter = odcs_server.get("delimiter")
+        server.endpointUrl = odcs_server.get("endpointUrl")
+        server.location = odcs_server.get("location")
+        server.account = odcs_server.get("account")
+        server.database = odcs_server.get("database")
+        server.schema_ = odcs_server.get("schema")
+        server.host = odcs_server.get("host")
+        server.port = odcs_server.get("port")
+        server.catalog = odcs_server.get("catalog")
+        server.topic = odcs_server.get("topic")
+        server.http_path = odcs_server.get("http_path")
+        server.token = odcs_server.get("token")
+        server.dataProductId = odcs_server.get("dataProductId")
+        server.outputPortId = odcs_server.get("outputPortId")
+        server.driver = odcs_server.get("driver")
+        server.roles = odcs_server.get("roles")
+
+        servers[server_name] = server
+    return servers
+
+
+def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None:
+    if odcs_contract.get("description") is None:
+        return None
+    if (
+        odcs_contract.get("description").get("usage") is not None
+        or odcs_contract.get("description").get("limitations") is not None
+        or odcs_contract.get("price") is not None
+    ):
+        terms = Terms()
+        if odcs_contract.get("description").get("usage") is not None:
+            terms.usage = odcs_contract.get("description").get("usage")
+        if odcs_contract.get("description").get("limitations") is not None:
+            terms.limitations = odcs_contract.get("description").get("limitations")
+        if odcs_contract.get("price") is not None:
+            terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}"
+
+        return terms
+    else:
+        return None
+
+
+def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel:
+    # find the two properties we can map (based on the examples)
+    sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else []
+    availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None)
+    retention = next((p for p in sla_properties if p["property"] == "retention"), None)
+
+    if availability is not None or retention is not None:
+        servicelevel = ServiceLevel()
+
+        if availability is not None:
+            value = availability.get("value")
+            if isinstance(value, datetime.datetime):
+                value = value.isoformat()
+            servicelevel.availability = Availability(description=value)
+
+        if retention is not None:
+            servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}")
+
+        return servicelevel
+    else:
+        return None
+
+
+def get_server_type(odcs_contract: Dict[str, Any]) -> str | None:
+    servers = import_servers(odcs_contract)
+    if servers is None or len(servers) == 0:
+        return None
+    # get first server from map
+    server = next(iter(servers.values()))
+    return server.type
+
+
+def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
+    custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties"))
+
+    odcs_schemas = odcs_contract.get("schema") if odcs_contract.get("schema") is not None else []
+    result = {}
+
+    for odcs_schema in odcs_schemas:
+        schema_name = odcs_schema.get("name")
+        schema_physical_name = odcs_schema.get("physicalName")
+        schema_description = odcs_schema.get("description") if odcs_schema.get("description") is not None else ""
+        model_name = schema_physical_name if schema_physical_name is not None else schema_name
+        model = Model(description=" ".join(schema_description.splitlines()), type="table")
+        model.fields = import_fields(
+            odcs_schema.get("properties"), custom_type_mappings, server_type=get_server_type(odcs_contract)
+        )
+        model.title = schema_name
+        if odcs_schema.get("dataGranularityDescription") is not None:
+            model.config = {"dataGranularityDescription": odcs_schema.get("dataGranularityDescription")}
+        result[model_name] = model
+
+    return result
+
+
+def import_field_config(odcs_property: Dict[str, Any], server_type=None) -> Dict[str, Any]:
+    config = {}
+    if odcs_property.get("criticalDataElement") is not None:
+        config["criticalDataElement"] = odcs_property.get("criticalDataElement")
+    if odcs_property.get("encryptedName") is not None:
+        config["encryptedName"] = odcs_property.get("encryptedName")
+    if odcs_property.get("partitionKeyPosition") is not None:
+        config["partitionKeyPosition"] = odcs_property.get("partitionKeyPosition")
+    if odcs_property.get("partitioned") is not None:
+        config["partitioned"] = odcs_property.get("partitioned")
+
+    if odcs_property.get("customProperties") is not None and isinstance(odcs_property.get("customProperties"), list):
+        for item in odcs_property.get("customProperties"):
+            config[item["property"]] = item["value"]
+
+    physical_type = odcs_property.get("physicalType")
+    if physical_type is not None:
+        if server_type == "postgres" or server_type == "postgresql":
+            config["postgresType"] = physical_type
+        elif server_type == "bigquery":
+            config["bigqueryType"] = physical_type
+        elif server_type == "snowflake":
+            config["snowflakeType"] = physical_type
+        elif server_type == "redshift":
+            config["redshiftType"] = physical_type
+        elif server_type == "sqlserver":
+            config["sqlserverType"] = physical_type
+        elif server_type == "databricksType":
+            config["databricksType"] = physical_type
+        else:
+            config["physicalType"] = physical_type
+
+    return config
+
+
+def has_composite_primary_key(odcs_properties) -> bool:
+    primary_keys = [prop for prop in odcs_properties if prop.get("primaryKey") is not None and prop.get("primaryKey")]
+    return len(primary_keys) > 1
+
+
+def import_fields(
+    odcs_properties: Dict[str, Any], custom_type_mappings: Dict[str, str], server_type
+) -> Dict[str, Field]:
+    logger = logging.getLogger(__name__)
+    result = {}
+
+    if odcs_properties is None:
+        return result
+
+    for odcs_property in odcs_properties:
+        mapped_type = map_type(odcs_property.get("logicalType"), custom_type_mappings)
+        if mapped_type is not None:
+            property_name = odcs_property["name"]
+            description = odcs_property.get("description") if odcs_property.get("description") is not None else None
+            field = Field(
+                description=" ".join(description.splitlines()) if description is not None else None,
+                type=mapped_type,
+                title=odcs_property.get("businessName"),
+                required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False,
+                primary=odcs_property.get("primaryKey")
+                if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None
+                else False,
+                unique=odcs_property.get("unique"),
+                examples=odcs_property.get("examples") if odcs_property.get("examples") is not None else None,
+                classification=odcs_property.get("classification")
+                if odcs_property.get("classification") is not None
+                else "",
+                tags=odcs_property.get("tags") if odcs_property.get("tags") is not None else None,
+                quality=odcs_property.get("quality") if odcs_property.get("quality") is not None else [],
+                config=import_field_config(odcs_property, server_type),
+            )
+            result[property_name] = field
+        else:
+            logger.info(
+                f"Can't map {odcs_property.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{odcs_property.get('logicalName')}' that defines your expected type as the 'value'"
+            )
+
+    return result
+
+
+def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
+    t = odcs_type.lower()
+    if t in DATACONTRACT_TYPES:
+        return t
+    elif custom_mappings.get(t) is not None:
+        return custom_mappings.get(t)
+    else:
+        return None
+
+
+def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]:
+    result = {}
+    if odcs_custom_properties is not None:
+        for prop in odcs_custom_properties:
+            if prop["property"].startswith("dc_mapping_"):
+                odcs_type_name = prop["property"].substring(11)
+                datacontract_type = prop["value"]
+                result[odcs_type_name] = datacontract_type
+
+    return result
+
+
+def import_tags(odcs_contract) -> List[str] | None:
+    if odcs_contract.get("tags") is None:
+        return None
+    return odcs_contract.get("tags")

datacontract/imports/parquet_importer.py

@@ -0,0 +1,81 @@
+import os.path
+
+import pyarrow
+from pyarrow import parquet
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Field,
+    Model,
+)
+from datacontract.model.exceptions import DataContractException
+
+
+class ParquetImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_parquet(data_contract_specification, source)
+
+
+def import_parquet(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    # use filename as schema name, remove .parquet suffix, avoid breaking the yaml output by replacing dots
+    schema_name = os.path.basename(source).removesuffix(".parquet").replace(".", "_")
+
+    fields: dict[str, Field] = {}
+
+    arrow_schema = parquet.read_schema(source)
+    for field_name in arrow_schema.names:
+        parquet_field = arrow_schema.field(field_name)
+
+        field = map_pyarrow_field_to_specification_field(parquet_field, "parquet")
+
+        if not parquet_field.nullable:
+            field.required = True
+
+        fields[field_name] = field
+
+    data_contract_specification.models[schema_name] = Model(fields=fields)
+
+    return data_contract_specification
+
+
+def map_pyarrow_field_to_specification_field(pyarrow_field: pyarrow.Field, file_format: str) -> Field:
+    if pyarrow.types.is_boolean(pyarrow_field.type):
+        return Field(type="boolean")
+    if pyarrow.types.is_int32(pyarrow_field.type):
+        return Field(type="int")
+    if pyarrow.types.is_int64(pyarrow_field.type):
+        return Field(type="long")
+    if pyarrow.types.is_integer(pyarrow_field.type):
+        return Field(type="number")
+    if pyarrow.types.is_float32(pyarrow_field.type):
+        return Field(type="float")
+    if pyarrow.types.is_float64(pyarrow_field.type):
+        return Field(type="double")
+    if pyarrow.types.is_decimal(pyarrow_field.type):
+        return Field(type="decimal", precision=pyarrow_field.type.precision, scale=pyarrow_field.type.scale)
+    if pyarrow.types.is_timestamp(pyarrow_field.type):
+        return Field(type="timestamp")
+    if pyarrow.types.is_date(pyarrow_field.type):
+        return Field(type="date")
+    if pyarrow.types.is_null(pyarrow_field.type):
+        return Field(type="null")
+    if pyarrow.types.is_binary(pyarrow_field.type):
+        return Field(type="bytes")
+    if pyarrow.types.is_string(pyarrow_field.type):
+        return Field(type="string")
+    if pyarrow.types.is_map(pyarrow_field.type) or pyarrow.types.is_dictionary(pyarrow_field.type):
+        return Field(type="map")
+    if pyarrow.types.is_struct(pyarrow_field.type):
+        return Field(type="struct")
+    if pyarrow.types.is_list(pyarrow_field.type):
+        return Field(type="array")
+
+    raise DataContractException(
+        type="schema",
+        name=f"Parse {file_format} schema",
+        reason=f"{pyarrow_field.type} currently not supported.",
+        engine="datacontract",
+    )

datacontract/imports/spark_importer.py

@@ -1,9 +1,10 @@
 from pyspark.sql import DataFrame, SparkSession, types
+
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
-    Model,
     Field,
+    Model,
     Server,
 )
 

datacontract/imports/sql_importer.py

@@ -1,7 +1,7 @@
 from simple_ddl_parser import parse_from_file
 
 from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
 
 class SqlImporter(Importer):

datacontract/imports/unity_importer.py

@@ -2,13 +2,13 @@ import json
 import os
 from typing import List, Optional
 
-from pyspark.sql import types
 from databricks.sdk import WorkspaceClient
-from databricks.sdk.service.catalog import TableInfo, ColumnInfo
+from databricks.sdk.service.catalog import ColumnInfo, TableInfo
+from pyspark.sql import types
 
 from datacontract.imports.importer import Importer
 from datacontract.imports.spark_importer import _field_from_struct_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 from datacontract.model.exceptions import DataContractException
 
 

datacontract/integration/datamesh_manager.py

@@ -23,7 +23,7 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
             )
 
         if run.dataContractId is None:
-            raise Exception("Cannot publish run results, as data contract ID is unknown")
+            raise Exception("Cannot publish run results for unknown data contract ID")
 
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
         request_body = run.model_dump_json()

datacontract/integration/opentelemetry.py

@@ -12,7 +12,6 @@ from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExpo
 
 from datacontract.model.run import Run
 
-
 # Publishes metrics of a test run.
 # Metric contains the values:
 # 0 == test run passed,

datacontract/lint/lint.py

@@ -1,9 +1,10 @@
 import abc
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Sequence, Any, cast
+from typing import Any, Sequence, cast
 
 from datacontract.model.run import Check
+
 from ..model.data_contract_specification import DataContractSpecification
 
 """This module contains linter definitions for linting a data contract.

datacontract/lint/linters/description_linter.py

@@ -1,4 +1,5 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/example_model_linter.py

@@ -5,6 +5,7 @@ import json
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Example
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/field_pattern_linter.py

@@ -1,6 +1,7 @@
 import re
 
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/field_reference_linter.py

@@ -1,4 +1,5 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/notice_period_linter.py

@@ -1,6 +1,7 @@
 import re
 
 from datacontract.model.data_contract_specification import DataContractSpecification
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/quality_schema_linter.py

@@ -1,6 +1,7 @@
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Model
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/linters/valid_constraints_linter.py

@@ -1,4 +1,5 @@
 from datacontract.model.data_contract_specification import DataContractSpecification, Field
+
 from ..lint import Linter, LinterResult
 
 

datacontract/lint/resolve.py

@@ -5,11 +5,13 @@ import fastjsonschema
 import yaml
 from fastjsonschema import JsonSchemaValueException
 
-from datacontract.lint.files import read_file
+from datacontract.imports.odcs_v3_importer import import_odcs_v3_from_str
+from datacontract.lint.resources import read_resource
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
 from datacontract.model.data_contract_specification import DataContractSpecification, Definition, Quality
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.odcs import is_open_data_contract_standard
 
 
 def resolve_data_contract(
@@ -41,10 +43,7 @@ def resolve_data_contract(
 def resolve_data_contract_from_location(
     location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
-    if location.startswith("http://") or location.startswith("https://"):
-        data_contract_str = fetch_resource(location)
-    else:
-        data_contract_str = read_file(location)
+    data_contract_str = read_resource(location)
     return _resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
 
 
@@ -196,10 +195,16 @@ def _get_quality_ref_file(quality_spec: str | object) -> str | object:
 def _resolve_data_contract_from_str(
     data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
-    data_contract_yaml_dict = _to_yaml(data_contract_str)
-    _validate(data_contract_yaml_dict, schema_location)
+    yaml_dict = _to_yaml(data_contract_str)
+
+    if is_open_data_contract_standard(yaml_dict):
+        # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
+        data_contract_specification = DataContractSpecification(dataContractSpecification="1.1.0")
+        return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str)
 
-    spec = DataContractSpecification(**data_contract_yaml_dict)
+    _validate_data_contract_specification_schema(yaml_dict, schema_location)
+    data_contract_specification = yaml_dict
+    spec = DataContractSpecification(**data_contract_specification)
 
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
@@ -224,7 +229,7 @@ def _to_yaml(data_contract_str):
         )
 
 
-def _validate(data_contract_yaml, schema_location: str = None):
+def _validate_data_contract_specification_schema(data_contract_yaml, schema_location: str = None):
     schema = fetch_schema(schema_location)
     try:
         fastjsonschema.validate(schema, data_contract_yaml)

datacontract/lint/resources.py

@@ -0,0 +1,21 @@
+from datacontract.lint.files import read_file
+from datacontract.lint.urls import fetch_resource
+
+
+def read_resource(location: str) -> str:
+    """
+    Read a resource from a given location.
+
+    If the location is a URL, fetch the resource from the web. API-Keys are supported.
+    Otherwise, read the resource from a local file.
+
+    Args:
+        location (str): The location of the resource, either a URL or a file path.
+
+    Returns:
+        str: The content of the resource.
+    """
+    if location.startswith("http://") or location.startswith("https://"):
+        return fetch_resource(location)
+    else:
+        return read_file(location)

datacontract/lint/schema.py

@@ -1,6 +1,6 @@
 import json
 import os
-from typing import Dict, Any
+from typing import Any, Dict
 
 import requests
 

datacontract/lint/urls.py

@@ -1,4 +1,5 @@
 import os
+from urllib.parse import urlparse
 
 import requests
 
@@ -25,7 +26,8 @@ def fetch_resource(url: str):
 
 
 def _set_api_key(headers, url):
-    if ".datamesh-manager.com/" in url:
+    hostname = urlparse(url).hostname
+    if hostname == "datamesh-manager.com" or hostname.endswith(".datamesh-manager.com"):
         datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
         if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
             print("Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
@@ -37,7 +39,7 @@ def _set_api_key(headers, url):
                 result="error",
             )
         headers["x-api-key"] = datamesh_manager_api_key
-    elif ".datacontract-manager.com/" in url:
+    elif hostname == "datacontract-manager.com" or hostname.endswith(".datacontract-manager.com"):
         datacontract_manager_api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
         if datacontract_manager_api_key is None or datacontract_manager_api_key == "":
             print("Error: Data Contract Manager API Key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.")