datacontract-cli 0.10.12__py3-none-any.whl → 0.10.14__py3-none-any.whl

datacontract/export/odcs_v3_exporter.py

@@ -0,0 +1,294 @@
+from typing import Dict
+
+import yaml
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+
+
+class OdcsV3Exporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_odcs_v3_yaml(data_contract)
+
+
+def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
+    odcs = {
+        "apiVersion": "v3.0.0",
+        "kind": "DataContract",
+        "id": data_contract_spec.id,
+        "name": data_contract_spec.info.title,
+        "version": data_contract_spec.info.version,
+        "domain": data_contract_spec.info.owner,
+        "status": data_contract_spec.info.status,
+    }
+
+    if data_contract_spec.terms is not None:
+        odcs["description"] = {
+            "purpose": data_contract_spec.terms.description.strip()
+            if data_contract_spec.terms.description is not None
+            else None,
+            "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
+            "limitations": data_contract_spec.terms.limitations.strip()
+            if data_contract_spec.terms.limitations is not None
+            else None,
+        }
+
+    odcs["schema"] = []
+    for model_key, model_value in data_contract_spec.models.items():
+        odcs_schema = to_odcs_schema(model_key, model_value)
+        odcs["schema"].append(odcs_schema)
+
+    if data_contract_spec.servicelevels is not None:
+        slas = []
+        if data_contract_spec.servicelevels.availability is not None:
+            slas.append(
+                {
+                    "property": "generalAvailability",
+                    "value": data_contract_spec.servicelevels.availability.description,
+                }
+            )
+        if data_contract_spec.servicelevels.retention is not None:
+            slas.append({"property": "retention", "value": data_contract_spec.servicelevels.retention.period})
+
+        if len(slas) > 0:
+            odcs["slaProperties"] = slas
+
+    if data_contract_spec.info.contact is not None:
+        support = []
+        if data_contract_spec.info.contact.email is not None:
+            support.append(
+                {
+                    "channel": "email",
+                    "url": "mailto:" + data_contract_spec.info.contact.email,
+                }
+            )
+        if data_contract_spec.info.contact.url is not None:
+            support.append(
+                {
+                    "channel": "other",
+                    "url": data_contract_spec.info.contact.url,
+                }
+            )
+        if len(support) > 0:
+            odcs["support"] = support
+
+    if data_contract_spec.servers is not None and len(data_contract_spec.servers) > 0:
+        servers = []
+
+        for server_key, server_value in data_contract_spec.servers.items():
+            server_dict = {}
+            server_dict["server"] = server_key
+            if server_value.type is not None:
+                server_dict["type"] = server_value.type
+            if server_value.environment is not None:
+                server_dict["environment"] = server_value.environment
+            if server_value.account is not None:
+                server_dict["account"] = server_value.account
+            if server_value.database is not None:
+                server_dict["database"] = server_value.database
+            if server_value.schema_ is not None:
+                server_dict["schema"] = server_value.schema_
+            if server_value.format is not None:
+                server_dict["format"] = server_value.format
+            if server_value.project is not None:
+                server_dict["project"] = server_value.project
+            if server_value.dataset is not None:
+                server_dict["dataset"] = server_value.dataset
+            if server_value.path is not None:
+                server_dict["path"] = server_value.path
+            if server_value.delimiter is not None:
+                server_dict["delimiter"] = server_value.delimiter
+            if server_value.endpointUrl is not None:
+                server_dict["endpointUrl"] = server_value.endpointUrl
+            if server_value.location is not None:
+                server_dict["location"] = server_value.location
+            if server_value.host is not None:
+                server_dict["host"] = server_value.host
+            if server_value.port is not None:
+                server_dict["port"] = server_value.port
+            if server_value.catalog is not None:
+                server_dict["catalog"] = server_value.catalog
+            if server_value.topic is not None:
+                server_dict["topic"] = server_value.topic
+            if server_value.http_path is not None:
+                server_dict["http_path"] = server_value.http_path
+            if server_value.token is not None:
+                server_dict["token"] = server_value.token
+            if server_value.driver is not None:
+                server_dict["driver"] = server_value.driver
+            if server_value.roles is not None:
+                server_dict["roles"] = [
+                    {"name": role.name, "description": role.description} for role in server_value.roles
+                ]
+            servers.append(server_dict)
+
+        if len(servers) > 0:
+            odcs["servers"] = servers
+
+    odcs["customProperties"] = []
+    if data_contract_spec.info.model_extra is not None:
+        for key, value in data_contract_spec.info.model_extra.items():
+            odcs["customProperties"].append({"property": key, "value": value})
+    if len(odcs["customProperties"]) == 0:
+        del odcs["customProperties"]
+
+    return yaml.dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
+
+
+def to_odcs_schema(model_key, model_value: Model) -> dict:
+    odcs_table = {
+        "name": model_key,
+        "physicalName": model_key,
+        "logicalType": "object",
+        "physicalType": model_value.type,
+    }
+    if model_value.description is not None:
+        odcs_table["description"] = model_value.description
+    properties = to_properties(model_value.fields)
+    if properties:
+        odcs_table["properties"] = properties
+
+    odcs_table["customProperties"] = []
+    if model_value.model_extra is not None:
+        for key, value in model_value.model_extra.items():
+            odcs_table["customProperties"].append({"property": key, "value": value})
+    if len(odcs_table["customProperties"]) == 0:
+        del odcs_table["customProperties"]
+
+    return odcs_table
+
+
+def to_properties(fields: Dict[str, Field]) -> list:
+    properties = []
+    for field_name, field in fields.items():
+        property = to_property(field_name, field)
+        properties.append(property)
+    return properties
+
+
+def to_logical_type(type: str) -> str | None:
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "string"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "date"
+    if type.lower() in ["timestamp_ntz"]:
+        return "date"
+    if type.lower() in ["date"]:
+        return "date"
+    if type.lower() in ["time"]:
+        return "string"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        return "number"
+    if type.lower() in ["float", "double"]:
+        return "number"
+    if type.lower() in ["integer", "int", "long", "bigint"]:
+        return "integer"
+    if type.lower() in ["boolean"]:
+        return "boolean"
+    if type.lower() in ["object", "record", "struct"]:
+        return "object"
+    if type.lower() in ["bytes"]:
+        return "array"
+    if type.lower() in ["array"]:
+        return "array"
+    if type.lower() in ["null"]:
+        return None
+    return None
+
+
+def to_physical_type(type: str) -> str | None:
+    # TODO: to we need to do a server mapping here?
+    return type
+
+
+def to_property(field_name: str, field: Field) -> dict:
+    property = {"name": field_name}
+    if field.title is not None:
+        property["businessName"] = field.title
+    if field.type is not None:
+        property["logicalType"] = to_logical_type(field.type)
+        property["physicalType"] = to_physical_type(field.type)
+    if field.description is not None:
+        property["description"] = field.description
+    if field.required is not None:
+        property["isNullable"] = not field.required
+    if field.unique is not None:
+        property["isUnique"] = field.unique
+    if field.classification is not None:
+        property["classification"] = field.classification
+    if field.examples is not None:
+        property["examples"] = field.examples
+    if field.example is not None:
+        property["examples"] = [field.example]
+
+    property["customProperties"] = []
+    if field.model_extra is not None:
+        for key, value in field.model_extra.items():
+            property["customProperties"].append({"property": key, "value": value})
+    if field.pii is not None:
+        property["customProperties"].append({"property": "pii", "value": field.pii})
+    if property.get("customProperties") is not None and len(property["customProperties"]) == 0:
+        del property["customProperties"]
+
+    property["tags"] = []
+    if field.tags is not None:
+        property["tags"].extend(field.tags)
+    if not property["tags"]:
+        del property["tags"]
+
+    property["logicalTypeOptions"] = {}
+    if field.minLength is not None:
+        property["logicalTypeOptions"]["minLength"] = field.minLength
+    if field.maxLength is not None:
+        property["logicalTypeOptions"]["maxLength"] = field.maxLength
+    if field.pattern is not None:
+        property["logicalTypeOptions"]["pattern"] = field.pattern
+    if field.minimum is not None:
+        property["logicalTypeOptions"]["minimum"] = field.minimum
+    if field.maximum is not None:
+        property["logicalTypeOptions"]["maximum"] = field.maximum
+    if field.exclusiveMinimum is not None:
+        property["logicalTypeOptions"]["exclusiveMinimum"] = field.exclusiveMinimum
+    if field.exclusiveMaximum is not None:
+        property["logicalTypeOptions"]["exclusiveMaximum"] = field.exclusiveMaximum
+    if property["logicalTypeOptions"] == {}:
+        del property["logicalTypeOptions"]
+
+    if field.quality is not None:
+        quality_property = []
+        for quality in field.quality:
+            quality_dict = {"type": quality.type}
+            if quality.description is not None:
+                quality_dict["description"] = quality.description
+            if quality.query is not None:
+                quality_dict["query"] = quality.query
+            # dialect is not supported in v3.0.0
+            if quality.mustBe is not None:
+                quality_dict["mustBe"] = quality.mustBe
+            if quality.mustNotBe is not None:
+                quality_dict["mustNotBe"] = quality.mustNotBe
+            if quality.mustBeGreaterThan is not None:
+                quality_dict["mustBeGreaterThan"] = quality.mustBeGreaterThan
+            if quality.mustBeGreaterThanOrEqualTo is not None:
+                quality_dict["mustBeGreaterThanOrEqualTo"] = quality.mustBeGreaterThanOrEqualTo
+            if quality.mustBeLessThan is not None:
+                quality_dict["mustBeLessThan"] = quality.mustBeLessThan
+            if quality.mustBeLessThanOrEqualTo is not None:
+                quality_dict["mustBeLessThanOrEqualTo"] = quality.mustBeLessThanOrEqualTo
+            if quality.mustBeBetween is not None:
+                quality_dict["mustBeBetween"] = quality.mustBeBetween
+            if quality.mustNotBeBetween is not None:
+                quality_dict["mustNotBeBetween"] = quality.mustNotBeBetween
+            if quality.engine is not None:
+                quality_dict["engine"] = quality.engine
+            if quality.implementation is not None:
+                quality_dict["implementation"] = quality.implementation
+            quality_property.append(quality_dict)
+        if len(quality_property) > 0:
+            property["quality"] = quality_property
+
+    # todo enum
+
+    return property

datacontract/export/sodacl_converter.py

@@ -1,8 +1,11 @@
+from typing import List
+from venv import logger
+
 import yaml
 
-from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.export.exporter import Exporter
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Quality
 
 
 class SodaExporter(Exporter):
@@ -58,9 +61,14 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
             checks.append(check_field_regex(field_name, field.pattern, quote_field_name))
         if field.enum is not None and len(field.enum) > 0:
             checks.append(check_field_enum(field_name, field.enum, quote_field_name))
+        if field.quality is not None and len(field.quality) > 0:
+            checks.append(check_quality_list(model_key, field_name, field.quality))
         # TODO references: str = None
         # TODO format
 
+    if model_value.quality is not None and len(model_value.quality) > 0:
+        checks.append(check_quality_list(model_key, None, model_value.quality))
+
     checks_for_model_key = f"checks for {model_key}"
 
     if quote_field_name:
@@ -181,6 +189,78 @@ def check_field_regex(field_name, pattern, quote_field_name: bool = False):
     }
 
 
+def check_quality_list(model_name, field_name, quality_list: List[Quality]):
+    checks = {}
+
+    count = 0
+    for quality in quality_list:
+        if quality.type == "sql":
+            if field_name is None:
+                metric_name = f"{model_name}_{field_name}_quality_sql_{count}"
+            else:
+                metric_name = f"{model_name}_quality_sql_{count}"
+            threshold = to_sodacl_threshold(quality)
+            query = prepare_query(quality, model_name, field_name)
+            if query is None:
+                logger.warning(f"Quality check {metric_name} has no query")
+                continue
+            if threshold is None:
+                logger.warning(f"Quality check {metric_name} has no valid threshold")
+                continue
+            checks[f"{metric_name} {threshold}"] = {f"{metric_name} query": query}
+        count += 1
+
+    return checks
+
+
+def prepare_query(quality: Quality, model_name: str, field_name: str = None) -> str | None:
+    if quality.query is None:
+        return None
+    if quality.query == "":
+        return None
+
+    query = quality.query
+
+    query = query.replace("{model}", model_name)
+    query = query.replace("{table}", model_name)
+
+    if field_name is not None:
+        query = query.replace("{field}", field_name)
+        query = query.replace("{column}", field_name)
+
+    return query
+
+
+def to_sodacl_threshold(quality: Quality) -> str | None:
+    if quality.mustBe is not None:
+        return f"= {quality.mustBe}"
+    if quality.mustNotBe is not None:
+        return f"!= {quality.mustNotBe}"
+    if quality.mustBeGreaterThan is not None:
+        return f"> {quality.mustBeGreaterThan}"
+    if quality.mustBeGreaterThanOrEqualTo is not None:
+        return f">= {quality.mustBeGreaterThanOrEqualTo}"
+    if quality.mustBeLessThan is not None:
+        return f"< {quality.mustBeLessThan}"
+    if quality.mustBeLessThanOrEqualTo is not None:
+        return f"<= {quality.mustBeLessThanOrEqualTo}"
+    if quality.mustBeBetween is not None:
+        if len(quality.mustBeBetween) != 2:
+            logger.warning(
+                f"Quality check has invalid mustBeBetween, must have exactly 2 integers in an array: {quality.mustBeBetween}"
+            )
+            return None
+        return f"between {quality.mustBeBetween[0]} and {quality.mustBeBetween[1]}"
+    if quality.mustNotBeBetween is not None:
+        if len(quality.mustNotBeBetween) != 2:
+            logger.warning(
+                f"Quality check has invalid mustNotBeBetween, must have exactly 2 integers in an array: {quality.mustNotBeBetween}"
+            )
+            return None
+        return f"not between {quality.mustNotBeBetween[0]} and {quality.mustNotBeBetween[1]}"
+    return None
+
+
 def add_quality_checks(sodacl, data_contract_spec):
     if data_contract_spec.quality is None:
         return

datacontract/export/spark_converter.py

@@ -128,7 +128,9 @@ def to_data_type(field: Field) -> types.DataType:
     if field_type in ["string", "varchar", "text"]:
         return types.StringType()
     if field_type in ["number", "decimal", "numeric"]:
-        return types.DecimalType()
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return types.DecimalType(precision=precision, scale=scale)
     if field_type in ["integer", "int"]:
         return types.IntegerType()
     if field_type == "long":

datacontract/export/sql_type_converter.py

@@ -7,6 +7,8 @@ def convert_to_sql_type(field: Field, server_type: str) -> str:
         return convert_to_snowflake(field)
     elif server_type == "postgres":
         return convert_type_to_postgres(field)
+    elif server_type == "dataframe":
+        return convert_to_dataframe(field)
     elif server_type == "databricks":
         return convert_to_databricks(field)
     elif server_type == "local" or server_type == "s3":
@@ -108,6 +110,46 @@ def convert_type_to_postgres(field: Field) -> None | str:
     return None
 
 
+# dataframe data types:
+# https://spark.apache.org/docs/latest/sql-ref-datatypes.html
+def convert_to_dataframe(field: Field) -> None | str:
+    if field.config and "dataframeType" in field.config:
+        return field.config["dataframeType"]
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "STRING"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP_NTZ"
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "STRING"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "DECIMAL"
+    if type.lower() in ["float"]:
+        return "FLOAT"
+    if type.lower() in ["double"]:
+        return "DOUBLE"
+    if type.lower() in ["integer", "int"]:
+        return "INT"
+    if type.lower() in ["long", "bigint"]:
+        return "BIGINT"
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        return "STRUCT"
+    if type.lower() in ["bytes"]:
+        return "BINARY"
+    if type.lower() in ["array"]:
+        return "ARRAY"
+    return None
+
+
 # databricks data types:
 # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
 def convert_to_databricks(field: Field) -> None | str:
@@ -186,7 +228,7 @@ def convert_to_duckdb(field: Field) -> None | str:
         "time": "TIME",
         "timestamp": "TIMESTAMP WITH TIME ZONE",
         "timestamp_tz": "TIMESTAMP WITH TIME ZONE",
-        "timestamp_ntz": "DATETIME",
+        "timestamp_ntz": "TIMESTAMP",
     }
 
     # Convert simple mappings
@@ -269,16 +311,7 @@ def convert_type_to_sqlserver(field: Field) -> None | str:
 
 def convert_type_to_bigquery(field: Field) -> None | str:
     """Convert from supported datacontract types to equivalent bigquery types"""
-    field_type = field.type
-    if not field_type:
-        return None
-
-    # If provided sql-server config type, prefer it over default mapping
-    if bigquery_type := get_type_config(field, "bigqueryType"):
-        return bigquery_type
-
-    field_type = field_type.lower()
-    return map_type_to_bigquery(field_type, field.title)
+    return map_type_to_bigquery(field)
 
 
 def get_type_config(field: Field, config_attr: str) -> dict[str, str] | None:
@@ -290,25 +323,27 @@ def get_type_config(field: Field, config_attr: str) -> dict[str, str] | None:
 
 def convert_type_to_trino(field: Field) -> None | str:
     """Convert from supported datacontract types to equivalent trino types"""
-    field_type = field.type
+    if field.config and "trinoType" in field.config:
+        return field.config["trinoType"]
 
-    if field_type.lower() in ["string", "text", "varchar"]:
+    field_type = field.type.lower()
+    if field_type in ["string", "text", "varchar"]:
         return "varchar"
     # tinyint, smallint not supported by data contract
-    if field_type.lower() in ["number", "decimal", "numeric"]:
+    if field_type in ["number", "decimal", "numeric"]:
         # precision and scale not supported by data contract
         return "decimal"
-    if field_type.lower() in ["int", "integer"]:
+    if field_type in ["int", "integer"]:
         return "integer"
-    if field_type.lower() in ["long", "bigint"]:
+    if field_type in ["long", "bigint"]:
         return "bigint"
-    if field_type.lower() in ["float"]:
+    if field_type in ["float"]:
         return "real"
-    if field_type.lower() in ["timestamp", "timestamp_tz"]:
+    if field_type in ["timestamp", "timestamp_tz"]:
         return "timestamp(3) with time zone"
-    if field_type.lower() in ["timestamp_ntz"]:
+    if field_type in ["timestamp_ntz"]:
         return "timestamp(3)"
-    if field_type.lower() in ["bytes"]:
+    if field_type in ["bytes"]:
         return "varbinary"
-    if field_type.lower() in ["object", "record", "struct"]:
+    if field_type in ["object", "record", "struct"]:
         return "json"