datacontract-cli 0.10.13__py3-none-any.whl → 0.10.14__py3-none-any.whl

datacontract/imports/odcs_v3_importer.py

@@ -0,0 +1,309 @@
+import datetime
+import logging
+from typing import Any, Dict, List
+from venv import logger
+
+import yaml
+
+from datacontract.imports.importer import Importer
+from datacontract.lint.resources import read_resource
+from datacontract.model.data_contract_specification import (
+    Availability,
+    DataContractSpecification,
+    Info,
+    Model,
+    Field,
+    Retention,
+    Server,
+    ServiceLevel,
+    Terms,
+    DATACONTRACT_TYPES,
+)
+from datacontract.model.exceptions import DataContractException
+
+
+class OdcsImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        return import_odcs_v3(data_contract_specification, source)
+
+
+def import_odcs_v3(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    source_str = read_resource(source)
+    return import_odcs_v3_from_str(data_contract_specification, source_str)
+
+
+def import_odcs_v3_from_str(
+    data_contract_specification: DataContractSpecification, source_str: str
+) -> DataContractSpecification:
+    try:
+        odcs_contract = yaml.safe_load(source_str)
+    except Exception as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse ODCS contract",
+            reason=f"Failed to parse odcs contract from {source_str}",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    data_contract_specification.id = odcs_contract["id"]
+    data_contract_specification.info = import_info(odcs_contract)
+    data_contract_specification.servers = import_servers(odcs_contract)
+    data_contract_specification.terms = import_terms(odcs_contract)
+    data_contract_specification.servicelevels = import_servicelevels(odcs_contract)
+    data_contract_specification.models = import_models(odcs_contract)
+    data_contract_specification.tags = import_tags(odcs_contract)
+
+    return data_contract_specification
+
+
+def import_info(odcs_contract: Dict[str, Any]) -> Info:
+    info = Info()
+
+    info.title = odcs_contract.get("name") if odcs_contract.get("name") is not None else ""
+
+    if odcs_contract.get("version") is not None:
+        info.version = odcs_contract.get("version")
+
+    # odcs.description.purpose => datacontract.description
+    if odcs_contract.get("description") is not None and odcs_contract.get("description").get("purpose") is not None:
+        info.description = odcs_contract.get("description").get("purpose")
+
+    # odcs.domain => datacontract.owner
+    if odcs_contract.get("domain") is not None:
+        info.owner = odcs_contract.get("domain")
+
+    # add dataProduct as custom property
+    if odcs_contract.get("dataProduct") is not None:
+        info.dataProduct = odcs_contract.get("dataProduct")
+
+    # add tenant as custom property
+    if odcs_contract.get("tenant") is not None:
+        info.tenant = odcs_contract.get("tenant")
+
+    return info
+
+
+def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None:
+    if odcs_contract.get("servers") is None:
+        return None
+    servers = {}
+    for odcs_server in odcs_contract.get("servers"):
+        server_name = odcs_server.get("server")
+        if server_name is None:
+            logger.warning("Server name is missing, skipping server")
+            continue
+
+        server = Server()
+        server.type = odcs_server.get("type")
+        server.description = odcs_server.get("description")
+        server.environment = odcs_server.get("environment")
+        server.format = odcs_server.get("format")
+        server.project = odcs_server.get("project")
+        server.dataset = odcs_server.get("dataset")
+        server.path = odcs_server.get("path")
+        server.delimiter = odcs_server.get("delimiter")
+        server.endpointUrl = odcs_server.get("endpointUrl")
+        server.location = odcs_server.get("location")
+        server.account = odcs_server.get("account")
+        server.database = odcs_server.get("database")
+        server.schema_ = odcs_server.get("schema")
+        server.host = odcs_server.get("host")
+        server.port = odcs_server.get("port")
+        server.catalog = odcs_server.get("catalog")
+        server.topic = odcs_server.get("topic")
+        server.http_path = odcs_server.get("http_path")
+        server.token = odcs_server.get("token")
+        server.dataProductId = odcs_server.get("dataProductId")
+        server.outputPortId = odcs_server.get("outputPortId")
+        server.driver = odcs_server.get("driver")
+        server.roles = odcs_server.get("roles")
+
+        servers[server_name] = server
+    return servers
+
+
+def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None:
+    if odcs_contract.get("description") is None:
+        return None
+    if (
+        odcs_contract.get("description").get("usage") is not None
+        or odcs_contract.get("description").get("limitations") is not None
+        or odcs_contract.get("price") is not None
+    ):
+        terms = Terms()
+        if odcs_contract.get("description").get("usage") is not None:
+            terms.usage = odcs_contract.get("description").get("usage")
+        if odcs_contract.get("description").get("limitations") is not None:
+            terms.limitations = odcs_contract.get("description").get("limitations")
+        if odcs_contract.get("price") is not None:
+            terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}"
+
+        return terms
+    else:
+        return None
+
+
+def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel:
+    # find the two properties we can map (based on the examples)
+    sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else []
+    availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None)
+    retention = next((p for p in sla_properties if p["property"] == "retention"), None)
+
+    if availability is not None or retention is not None:
+        servicelevel = ServiceLevel()
+
+        if availability is not None:
+            value = availability.get("value")
+            if isinstance(value, datetime.datetime):
+                value = value.isoformat()
+            servicelevel.availability = Availability(description=value)
+
+        if retention is not None:
+            servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}")
+
+        return servicelevel
+    else:
+        return None
+
+
+def get_server_type(odcs_contract: Dict[str, Any]) -> str | None:
+    servers = import_servers(odcs_contract)
+    if servers is None or len(servers) == 0:
+        return None
+    # get first server from map
+    server = next(iter(servers.values()))
+    return server.type
+
+
+def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
+    custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties"))
+
+    odcs_schemas = odcs_contract.get("schema") if odcs_contract.get("schema") is not None else []
+    result = {}
+
+    for odcs_schema in odcs_schemas:
+        schema_name = odcs_schema.get("name")
+        schema_physical_name = odcs_schema.get("physicalName")
+        schema_description = odcs_schema.get("description") if odcs_schema.get("description") is not None else ""
+        model_name = schema_physical_name if schema_physical_name is not None else schema_name
+        model = Model(description=" ".join(schema_description.splitlines()), type="table")
+        model.fields = import_fields(
+            odcs_schema.get("properties"), custom_type_mappings, server_type=get_server_type(odcs_contract)
+        )
+        model.title = schema_name
+        if odcs_schema.get("dataGranularityDescription") is not None:
+            model.config = {"dataGranularityDescription": odcs_schema.get("dataGranularityDescription")}
+        result[model_name] = model
+
+    return result
+
+
+def import_field_config(odcs_property: Dict[str, Any], server_type=None) -> Dict[str, Any]:
+    config = {}
+    if odcs_property.get("criticalDataElement") is not None:
+        config["criticalDataElement"] = odcs_property.get("criticalDataElement")
+    if odcs_property.get("encryptedName") is not None:
+        config["encryptedName"] = odcs_property.get("encryptedName")
+    if odcs_property.get("partitionKeyPosition") is not None:
+        config["partitionKeyPosition"] = odcs_property.get("partitionKeyPosition")
+    if odcs_property.get("partitioned") is not None:
+        config["partitioned"] = odcs_property.get("partitioned")
+
+    if odcs_property.get("customProperties") is not None and isinstance(odcs_property.get("customProperties"), list):
+        for item in odcs_property.get("customProperties"):
+            config[item["property"]] = item["value"]
+
+    physical_type = odcs_property.get("physicalType")
+    if physical_type is not None:
+        if server_type == "postgres" or server_type == "postgresql":
+            config["postgresType"] = physical_type
+        elif server_type == "bigquery":
+            config["bigqueryType"] = physical_type
+        elif server_type == "snowflake":
+            config["snowflakeType"] = physical_type
+        elif server_type == "redshift":
+            config["redshiftType"] = physical_type
+        elif server_type == "sqlserver":
+            config["sqlserverType"] = physical_type
+        elif server_type == "databricksType":
+            config["databricksType"] = physical_type
+        else:
+            config["physicalType"] = physical_type
+
+    return config
+
+
+def has_composite_primary_key(odcs_properties) -> bool:
+    primary_keys = [prop for prop in odcs_properties if prop.get("primaryKey") is not None and prop.get("primaryKey")]
+    return len(primary_keys) > 1
+
+
+def import_fields(
+    odcs_properties: Dict[str, Any], custom_type_mappings: Dict[str, str], server_type
+) -> Dict[str, Field]:
+    logger = logging.getLogger(__name__)
+    result = {}
+
+    if odcs_properties is None:
+        return result
+
+    for odcs_property in odcs_properties:
+        mapped_type = map_type(odcs_property.get("logicalType"), custom_type_mappings)
+        if mapped_type is not None:
+            property_name = odcs_property["name"]
+            description = odcs_property.get("description") if odcs_property.get("description") is not None else None
+            field = Field(
+                description=" ".join(description.splitlines()) if description is not None else None,
+                type=mapped_type,
+                title=odcs_property.get("businessName"),
+                required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False,
+                primary=odcs_property.get("primaryKey")
+                if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None
+                else False,
+                unique=odcs_property.get("unique"),
+                examples=odcs_property.get("examples") if odcs_property.get("examples") is not None else None,
+                classification=odcs_property.get("classification")
+                if odcs_property.get("classification") is not None
+                else "",
+                tags=odcs_property.get("tags") if odcs_property.get("tags") is not None else None,
+                quality=odcs_property.get("quality") if odcs_property.get("quality") is not None else [],
+                config=import_field_config(odcs_property, server_type),
+            )
+            result[property_name] = field
+        else:
+            logger.info(
+                f"Can't map {odcs_property.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{odcs_property.get('logicalName')}' that defines your expected type as the 'value'"
+            )
+
+    return result
+
+
+def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
+    t = odcs_type.lower()
+    if t in DATACONTRACT_TYPES:
+        return t
+    elif custom_mappings.get(t) is not None:
+        return custom_mappings.get(t)
+    else:
+        return None
+
+
+def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]:
+    result = {}
+    if odcs_custom_properties is not None:
+        for prop in odcs_custom_properties:
+            if prop["property"].startswith("dc_mapping_"):
+                odcs_type_name = prop["property"].substring(11)
+                datacontract_type = prop["value"]
+                result[odcs_type_name] = datacontract_type
+
+    return result
+
+
+def import_tags(odcs_contract) -> List[str] | None:
+    if odcs_contract.get("tags") is None:
+        return None
+    return odcs_contract.get("tags")

datacontract/integration/datamesh_manager.py

@@ -23,7 +23,7 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
             )
 
         if run.dataContractId is None:
-            raise Exception("Cannot publish run results, as data contract ID is unknown")
+            raise Exception("Cannot publish run results for unknown data contract ID")
 
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
         request_body = run.model_dump_json()

datacontract/lint/resolve.py

@@ -5,11 +5,13 @@ import fastjsonschema
 import yaml
 from fastjsonschema import JsonSchemaValueException
 
-from datacontract.lint.files import read_file
+from datacontract.imports.odcs_v3_importer import import_odcs_v3_from_str
+from datacontract.lint.resources import read_resource
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
 from datacontract.model.data_contract_specification import DataContractSpecification, Definition, Quality
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.odcs import is_open_data_contract_standard
 
 
 def resolve_data_contract(
@@ -41,10 +43,7 @@ def resolve_data_contract(
 def resolve_data_contract_from_location(
     location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
-    if location.startswith("http://") or location.startswith("https://"):
-        data_contract_str = fetch_resource(location)
-    else:
-        data_contract_str = read_file(location)
+    data_contract_str = read_resource(location)
     return _resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
 
 
@@ -196,10 +195,16 @@ def _get_quality_ref_file(quality_spec: str | object) -> str | object:
 def _resolve_data_contract_from_str(
     data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
-    data_contract_yaml_dict = _to_yaml(data_contract_str)
-    _validate(data_contract_yaml_dict, schema_location)
+    yaml_dict = _to_yaml(data_contract_str)
+
+    if is_open_data_contract_standard(yaml_dict):
+        # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
+        data_contract_specification = DataContractSpecification(dataContractSpecification="0.9.3")
+        return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str)
 
-    spec = DataContractSpecification(**data_contract_yaml_dict)
+    _validate_data_contract_specification_schema(yaml_dict, schema_location)
+    data_contract_specification = yaml_dict
+    spec = DataContractSpecification(**data_contract_specification)
 
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
@@ -224,7 +229,7 @@ def _to_yaml(data_contract_str):
         )
 
 
-def _validate(data_contract_yaml, schema_location: str = None):
+def _validate_data_contract_specification_schema(data_contract_yaml, schema_location: str = None):
     schema = fetch_schema(schema_location)
     try:
         fastjsonschema.validate(schema, data_contract_yaml)

datacontract/lint/resources.py

@@ -0,0 +1,21 @@
+from datacontract.lint.files import read_file
+from datacontract.lint.urls import fetch_resource
+
+
+def read_resource(location: str) -> str:
+    """
+    Read a resource from a given location.
+
+    If the location is a URL, fetch the resource from the web. API-Keys are supported.
+    Otherwise, read the resource from a local file.
+
+    Args:
+        location (str): The location of the resource, either a URL or a file path.
+
+    Returns:
+        str: The content of the resource.
+    """
+    if location.startswith("http://") or location.startswith("https://"):
+        return fetch_resource(location)
+    else:
+        return read_file(location)

datacontract/lint/urls.py

@@ -1,4 +1,5 @@
 import os
+from urllib.parse import urlparse
 
 import requests
 
@@ -25,7 +26,8 @@ def fetch_resource(url: str):
 
 
 def _set_api_key(headers, url):
-    if ".datamesh-manager.com/" in url:
+    hostname = urlparse(url).hostname
+    if hostname == "datamesh-manager.com" or hostname.endswith(".datamesh-manager.com"):
         datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
         if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
             print("Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
@@ -37,7 +39,7 @@ def _set_api_key(headers, url):
                 result="error",
             )
         headers["x-api-key"] = datamesh_manager_api_key
-    elif ".datacontract-manager.com/" in url:
+    elif hostname == "datacontract-manager.com" or hostname.endswith(".datacontract-manager.com"):
         datacontract_manager_api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
         if datacontract_manager_api_key is None or datacontract_manager_api_key == "":
             print("Error: Data Contract Manager API Key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.")

datacontract/model/data_contract_specification.py

@@ -4,6 +4,32 @@ from typing import List, Dict, Optional, Any
 import pydantic as pyd
 import yaml
 
+DATACONTRACT_TYPES = [
+    "string",
+    "text",
+    "varchar",
+    "number",
+    "decimal",
+    "numeric",
+    "int",
+    "integer",
+    "long",
+    "bigint",
+    "float",
+    "double",
+    "boolean",
+    "timestamp",
+    "timestamp_tz",
+    "timestamp_ntz",
+    "date",
+    "array",
+    "bytes",
+    "object",
+    "record",
+    "struct",
+    "null",
+]
+
 
 class Contact(pyd.BaseModel):
     name: str = None
@@ -15,6 +41,14 @@ class Contact(pyd.BaseModel):
     )
 
 
+class ServerRole(pyd.BaseModel):
+    name: str = None
+    description: str = None
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
+
+
 class Server(pyd.BaseModel):
     type: str = None
     description: str = None
@@ -38,6 +72,7 @@ class Server(pyd.BaseModel):
     dataProductId: str = None
     outputPortId: str = None
     driver: str = None
+    roles: List[ServerRole] = None
 
     model_config = pyd.ConfigDict(
         extra="allow",
@@ -83,19 +118,40 @@ class Definition(pyd.BaseModel):
     )
 
 
+class Quality(pyd.BaseModel):
+    type: str = None
+    description: str = None
+    query: str = None
+    dialect: str = None
+    mustBe: int = None
+    mustNotBe: int = None
+    mustBeGreaterThan: int = None
+    mustBeGreaterThanOrEqualTo: int = None
+    mustBeLessThan: int = None
+    mustBeLessThanOrEqualTo: int = None
+    mustBeBetween: List[int] = None
+    mustNotBeBetween: List[int] = None
+    engine: str = None
+    implementation: str | Dict[str, Any] = None
+
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
+
+
 class Field(pyd.BaseModel):
     ref: str = pyd.Field(default=None, alias="$ref")
     ref_obj: Definition = pyd.Field(default=None, exclude=True)
-    title: str = None
+    title: str | None = None
     type: str = None
     format: str = None
     required: bool = None
     primary: bool = None
-    unique: bool = None
+    unique: bool | None = None
     references: str = None
-    description: str = None
-    pii: bool = None
-    classification: str = None
+    description: str | None = None
+    pii: bool | None = None
+    classification: str | None = None
     pattern: str = None
     minLength: int = None
     maxLength: int = None
@@ -103,8 +159,8 @@ class Field(pyd.BaseModel):
     exclusiveMinimum: int = None
     maximum: int = None
     exclusiveMaximum: int = None
-    enum: List[str] = []
-    tags: List[str] = []
+    enum: List[str] | None = []
+    tags: List[str] | None = []
     links: Dict[str, str] = {}
     fields: Dict[str, "Field"] = {}
     items: "Field" = None
@@ -113,7 +169,9 @@ class Field(pyd.BaseModel):
     precision: int = None
     scale: int = None
     example: str = None
-    config: Dict[str, Any] = None
+    examples: List[Any] | None = None
+    quality: List[Quality] | None = []
+    config: Dict[str, Any] | None = None
 
     model_config = pyd.ConfigDict(
         extra="allow",
@@ -126,7 +184,13 @@ class Model(pyd.BaseModel):
     namespace: Optional[str] = None
     title: Optional[str] = None
     fields: Dict[str, Field] = {}
+    quality: List[Quality] | None = []
     config: Dict[str, Any] = None
+    tags: List[str] | None = None
+
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
 
 
 class Info(pyd.BaseModel):

datacontract/model/odcs.py

@@ -0,0 +1,11 @@
+def is_open_data_contract_standard(odcs: dict) -> bool:
+    """
+    Check if the given dictionary is an OpenDataContractStandard.
+
+    Args:
+        odcs (dict): The dictionary to check.
+
+    Returns:
+        bool: True if the dictionary is an OpenDataContractStandard, False otherwise.
+    """
+    return odcs.get("kind") == "DataContract" and odcs.get("apiVersion", "").startswith("v3")