datacontract-cli 0.10.21__py3-none-any.whl → 0.10.22__py3-none-any.whl

datacontract/engines/data_contract_test.py

@@ -0,0 +1,51 @@
+import typing
+
+from datacontract.engines.data_contract_checks import create_checks
+
+if typing.TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
+from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
+    check_that_datacontract_contains_valid_server_configuration,
+)
+from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
+from datacontract.engines.soda.check_soda_execute import check_soda_execute
+from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum, Run
+
+
+def execute_data_contract_test(
+    data_contract_specification: DataContractSpecification,
+    run: Run,
+    server_name: str = None,
+    spark: "SparkSession" = None,
+):
+    if data_contract_specification.models is None or len(data_contract_specification.models) == 0:
+        raise DataContractException(
+            type="lint",
+            name="Check that data contract contains models",
+            result=ResultEnum.warning,
+            reason="Models block is missing. Skip executing tests.",
+            engine="datacontract",
+        )
+    check_that_datacontract_contains_valid_server_configuration(run, data_contract_specification, server_name)
+    if server_name:
+        server = data_contract_specification.servers.get(server_name)
+    else:
+        server_name = list(data_contract_specification.servers.keys())[0]
+        server = data_contract_specification.servers.get(server_name)
+    run.log_info(f"Running tests for data contract {data_contract_specification.id} with server {server_name}")
+    run.dataContractId = data_contract_specification.id
+    run.dataContractVersion = data_contract_specification.info.version
+    run.dataProductId = server.dataProductId
+    run.outputPortId = server.outputPortId
+    run.server = server_name
+
+    run.checks.extend(create_checks(data_contract_specification, server))
+
+    # TODO check server is supported type for nicer error messages
+    # TODO check server credentials are complete for nicer error messages
+    if server.format == "json" and server.type != "kafka":
+        check_jsonschema(run, data_contract_specification, server)
+    check_soda_execute(run, data_contract_specification, server, spark)

datacontract/engines/soda/check_soda_execute.py

@@ -1,4 +1,5 @@
 import logging
+import uuid
 
 from datacontract.engines.soda.connections.bigquery import to_bigquery_soda_configuration
 from datacontract.engines.soda.connections.databricks import to_databricks_soda_configuration
@@ -13,7 +14,7 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
 from datacontract.model.run import Check, Log, ResultEnum, Run
 
 
-def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
+def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark):
     from soda.common.config_helper import ConfigHelper
 
     ConfigHelper.get_instance().upsert_value("send_anonymous_usage_stats", False)
@@ -80,8 +81,8 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
             scan.set_data_source_name("datacontract-cli")
     elif server.type == "kafka":
         if spark is None:
-            spark = create_spark_session(tmp_dir)
-        read_kafka_topic(spark, data_contract, server, tmp_dir)
+            spark = create_spark_session()
+        read_kafka_topic(spark, data_contract, server)
         scan.add_spark_session(spark, data_source_name=server.type)
         scan.set_data_source_name(server.type)
     elif server.type == "sqlserver":
@@ -106,37 +107,34 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         run.log_warn(f"Server type {server.type} not yet supported by datacontract CLI")
         return
 
-    # Don't check types for json format, as they are checked with json schema
-    # Don't check types for avro format, as they are checked with avro schema
-    # Don't check types for csv format, as they are hard to detect
-    server_type = server.type
-    check_types = server.format != "json" and server.format != "csv" and server.format != "avro"
-
-    sodacl_yaml_str = to_sodacl_yaml(data_contract, server_type, check_types)
+    sodacl_yaml_str = to_sodacl_yaml(run)
     # print("sodacl_yaml_str:\n" + sodacl_yaml_str)
     scan.add_sodacl_yaml_str(sodacl_yaml_str)
 
     # Execute the scan
-    logging.info("Starting soda scan")
+    logging.info("Starting soda scan with checks:\n" + sodacl_yaml_str)
     scan.execute()
     logging.info("Finished soda scan")
 
     # pprint.PrettyPrinter(indent=2).pprint(scan.build_scan_results())
 
     scan_results = scan.get_scan_results()
-    for c in scan_results.get("checks"):
-        check = Check(
-            type="schema",
-            result=to_result(c),
-            reason=", ".join(c.get("outcomeReasons")),
-            name=c.get("name"),
-            model=c.get("table"),
-            field=c.get("column"),
-            engine="soda-core",
-            diagnostics=c.get("diagnostics"),
-        )
-        update_reason(check, c)
-        run.checks.append(check)
+    for scan_result in scan_results.get("checks"):
+        name = scan_result.get("name")
+        check = get_check(run, scan_result)
+        if check is None:
+            check = Check(
+                id=str(uuid.uuid4()),
+                category="custom",
+                type="custom",
+                name=name,
+                engine="soda-core",
+            )
+            run.checks.append(check)
+        check.result = to_result(scan_result)
+        check.reason = ", ".join(scan_result.get("outcomeReasons"))
+        check.diagnostics = scan_result.get("diagnostics")
+        update_reason(check, scan_result)
 
     for log in scan_results.get("logs"):
         run.logs.append(
@@ -152,8 +150,8 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         run.checks.append(
             Check(
                 type="general",
-                name="Execute quality checks",
-                result="warning",
+                name="Data Contract Tests",
+                result=ResultEnum.warning,
                 reason="Engine soda-core has errors. See the logs for details.",
                 engine="soda-core",
             )
@@ -161,14 +159,22 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         return
 
 
-def to_result(c) -> str:
+def get_check(run, scan_result) -> Check | None:
+    check_by_name = next((c for c in run.checks if c.key == scan_result.get("name")), None)
+    if check_by_name is not None:
+        return check_by_name
+
+    return None
+
+
+def to_result(c) -> ResultEnum:
     soda_outcome = c.get("outcome")
     if soda_outcome == "pass":
-        return "passed"
+        return ResultEnum.passed
     elif soda_outcome == "fail":
-        return "failed"
+        return ResultEnum.failed
     else:
-        return soda_outcome
+        return ResultEnum.unknown
 
 
 def update_reason(check, c):

datacontract/engines/soda/connections/kafka.py

@@ -1,12 +1,14 @@
+import atexit
 import logging
 import os
+import tempfile
 
 from datacontract.export.avro_converter import to_avro_schema_json
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Server
 from datacontract.model.exceptions import DataContractException
 
 
-def create_spark_session(tmp_dir: str):
+def create_spark_session():
     """Create and configure a Spark session."""
 
     try:
@@ -21,6 +23,9 @@ def create_spark_session(tmp_dir: str):
             original_exception=e,
         )
 
+    tmp_dir = tempfile.TemporaryDirectory(prefix="datacontract-cli-spark")
+    atexit.register(tmp_dir.cleanup)
+
     spark = (
         SparkSession.builder.appName("datacontract")
         .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse")
@@ -37,7 +42,7 @@ def create_spark_session(tmp_dir: str):
     return spark
 
 
-def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Server, tmp_dir):
+def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Server):
     """Read and process data from a Kafka topic based on the server configuration."""
 
     logging.info("Reading data from Kafka server %s topic %s", server.host, server.topic)
@@ -62,7 +67,7 @@ def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Se
                 type="test",
                 name="Configuring Kafka checks",
                 result="warning",
-                reason=f"Kafka format '{server.format}' is not supported. " f"Skip executing tests.",
+                reason=f"Kafka format '{server.format}' is not supported. Skip executing tests.",
                 engine="datacontract",
             )
 

datacontract/export/avro_converter.py

@@ -108,6 +108,8 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
     elif field.type in ["time"]:
         return "long"
     elif field.type in ["object", "record", "struct"]:
+        if field.config is not None and 'namespace' in field.config:
+            return to_avro_record(field_name ,field.fields ,field.description ,field.config['namespace'])
         return to_avro_record(field_name, field.fields, field.description, None)
     elif field.type in ["binary"]:
         return "bytes"

datacontract/export/exporter.py

@@ -25,8 +25,6 @@ class ExportFormat(str, Enum):
     dbt_sources = "dbt-sources"
     dbt_staging_sql = "dbt-staging-sql"
     odcs = "odcs"
-    odcs_v2 = "odcs_v2"
-    odcs_v3 = "odcs_v3"
     rdf = "rdf"
     avro = "avro"
     protobuf = "protobuf"

datacontract/export/exporter_factory.py

@@ -107,18 +107,6 @@ exporter_factory.register_lazy_exporter(
     class_name="JsonSchemaExporter",
 )
 
-exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v2,
-    module_path="datacontract.export.odcs_v2_exporter",
-    class_name="OdcsV2Exporter",
-)
-
-exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v3,
-    module_path="datacontract.export.odcs_v3_exporter",
-    class_name="OdcsV3Exporter",
-)
-
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.odcs,
     module_path="datacontract.export.odcs_v3_exporter",

datacontract/export/sodacl_converter.py

@@ -1,302 +1,30 @@
-from typing import List
-from venv import logger
-
 import yaml
 
+from datacontract.engines.data_contract_checks import create_checks
 from datacontract.export.exporter import Exporter
-from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Quality
+from datacontract.model.run import Run
 
 
 class SodaExporter(Exporter):
     def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
-        return to_sodacl_yaml(data_contract)
-
-
-def to_sodacl_yaml(
-    data_contract_spec: DataContractSpecification, server_type: str = None, check_types: bool = True
-) -> str:
-    try:
-        sodacl = {}
-        for model_key, model_value in data_contract_spec.models.items():
-            k, v = to_checks(model_key, model_value, server_type, check_types)
-            sodacl[k] = v
-        add_quality_checks(sodacl, data_contract_spec)
-        sodacl_yaml_str = yaml.dump(sodacl, default_flow_style=False, sort_keys=False)
-        return sodacl_yaml_str
-    except Exception as e:
-        return f"Error: {e}"
-
-
-def to_checks(model_key, model_value, server_type: str, check_types: bool):
-    checks = []
-    model_name = to_model_name(model_key, model_value, server_type)
-    fields = model_value.fields
-
-    quote_field_name = server_type in ["postgres", "sqlserver"]
-
-    for field_name, field in fields.items():
-        checks.append(check_field_is_present(field_name))
-        if check_types and field.type is not None:
-            sql_type = convert_to_sql_type(field, server_type)
-            checks.append(check_field_type(field_name, sql_type))
-        if field.required:
-            checks.append(check_field_required(field_name, quote_field_name))
-        if field.unique:
-            checks.append(check_field_unique(field_name, quote_field_name))
-        if field.minLength is not None:
-            checks.append(check_field_min_length(field_name, field.minLength, quote_field_name))
-        if field.maxLength is not None:
-            checks.append(check_field_max_length(field_name, field.maxLength, quote_field_name))
-        if field.minimum is not None:
-            checks.append(check_field_minimum(field_name, field.minimum, quote_field_name))
-        if field.maximum is not None:
-            checks.append(check_field_maximum(field_name, field.maximum, quote_field_name))
-        if field.exclusiveMinimum is not None:
-            checks.append(check_field_minimum(field_name, field.exclusiveMinimum, quote_field_name))
-            checks.append(check_field_not_equal(field_name, field.exclusiveMinimum, quote_field_name))
-        if field.exclusiveMaximum is not None:
-            checks.append(check_field_maximum(field_name, field.exclusiveMaximum, quote_field_name))
-            checks.append(check_field_not_equal(field_name, field.exclusiveMaximum, quote_field_name))
-        if field.pattern is not None:
-            checks.append(check_field_regex(field_name, field.pattern, quote_field_name))
-        if field.enum is not None and len(field.enum) > 0:
-            checks.append(check_field_enum(field_name, field.enum, quote_field_name))
-        if field.quality is not None and len(field.quality) > 0:
-            quality_list = check_quality_list(model_name, field_name, field.quality)
-            if (quality_list is not None) and len(quality_list) > 0:
-                checks.append(quality_list)
-        # TODO references: str = None
-        # TODO format
-
-    if model_value.quality is not None and len(model_value.quality) > 0:
-        quality_list = check_quality_list(model_name, None, model_value.quality)
-        if (quality_list is not None) and len(quality_list) > 0:
-            checks.append(quality_list)
-
-    checks_for_model_key = f"checks for {model_name}"
-
-    if quote_field_name:
-        checks_for_model_key = f'checks for "{model_name}"'
-
-    return checks_for_model_key, checks
-
-
-def to_model_name(model_key, model_value, server_type):
-    if server_type == "databricks":
-        if model_value.config is not None and "databricksTable" in model_value.config:
-            return model_value.config["databricksTable"]
-    if server_type == "snowflake":
-        if model_value.config is not None and "snowflakeTable" in model_value.config:
-            return model_value.config["snowflakeTable"]
-    if server_type == "sqlserver":
-        if model_value.config is not None and "sqlserverTable" in model_value.config:
-            return model_value.config["sqlserverTable"]
-    if server_type == "postgres" or server_type == "postgresql":
-        if model_value.config is not None and "postgresTable" in model_value.config:
-            return model_value.config["postgresTable"]
-    return model_key
-
-
-def check_field_is_present(field_name):
-    return {
-        "schema": {
-            "name": f"Check that field {field_name} is present",
-            "fail": {
-                "when required column missing": [field_name],
-            },
-        }
-    }
-
-
-def check_field_type(field_name: str, type: str):
-    return {
-        "schema": {
-            "name": f"Check that field {field_name} has type {type}",
-            "fail": {"when wrong column type": {field_name: type}},
-        }
-    }
-
-
-def check_field_required(field_name: str, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-
-    return {f"missing_count({field_name}) = 0": {"name": f"Check that required field {field_name} has no null values"}}
-
-
-def check_field_unique(field_name, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"duplicate_count({field_name}) = 0": {"name": f"Check that unique field {field_name} has no duplicate values"}
-    }
-
-
-def check_field_min_length(field_name, min_length, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a min length of {min_length}",
-            "valid min length": min_length,
-        }
-    }
-
-
-def check_field_max_length(field_name, max_length, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a max length of {max_length}",
-            "valid max length": max_length,
-        }
-    }
-
-
-def check_field_minimum(field_name, minimum, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a minimum of {minimum}",
-            "valid min": minimum,
-        }
-    }
-
-
-def check_field_maximum(field_name, maximum, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a maximum of {maximum}",
-            "valid max": maximum,
-        }
-    }
-
-
-def check_field_not_equal(field_name, value, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} is not equal to {value}",
-            "invalid values": [value],
-        }
-    }
-
-
-def check_field_enum(field_name, enum, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} only contains enum values {enum}",
-            "valid values": enum,
-        }
-    }
-
-
-def check_field_regex(field_name, pattern, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} matches regex pattern {pattern}",
-            "valid regex": pattern,
-        }
-    }
-
-
-def check_quality_list(model_name, field_name, quality_list: List[Quality]):
-    checks = {}
-
-    count = 0
-    for quality in quality_list:
-        if quality.type == "sql":
-            if field_name is None:
-                metric_name = f"{model_name}_quality_sql_{count}"
+        run = Run.create_run()
+        run.checks.extend(create_checks(data_contract, server))
+        return to_sodacl_yaml(run)
+
+
+def to_sodacl_yaml(run: Run) -> str:
+    sodacl_dict = {}
+    for run_check in run.checks:
+        if run_check.engine != "soda" or run_check.language != "sodacl":
+            continue
+        check_yaml_str = run_check.implementation
+        check_yaml_dict = yaml.safe_load(check_yaml_str)
+        for key, value in check_yaml_dict.items():
+            if key in sodacl_dict:
+                if isinstance(sodacl_dict[key], list) and isinstance(value, list):
+                    sodacl_dict[key].extend(value)
+                else:
+                    sodacl_dict[key].update(value)
             else:
-                metric_name = f"{model_name}_{field_name}_quality_sql_{count}"
-            threshold = to_sodacl_threshold(quality)
-            query = prepare_query(quality, model_name, field_name)
-            if query is None:
-                logger.warning(f"Quality check {metric_name} has no query")
-                continue
-            if threshold is None:
-                logger.warning(f"Quality check {metric_name} has no valid threshold")
-                continue
-            checks[f"{metric_name} {threshold}"] = {f"{metric_name} query": query}
-        count += 1
-
-    return checks
-
-
-def prepare_query(quality: Quality, model_name: str, field_name: str = None) -> str | None:
-    if quality.query is None:
-        return None
-    if quality.query == "":
-        return None
-
-    query = quality.query
-
-    query = query.replace("{model}", model_name)
-    query = query.replace("{table}", model_name)
-
-    if field_name is not None:
-        query = query.replace("{field}", field_name)
-        query = query.replace("{column}", field_name)
-
-    return query
-
-
-def to_sodacl_threshold(quality: Quality) -> str | None:
-    if quality.mustBe is not None:
-        return f"= {quality.mustBe}"
-    if quality.mustNotBe is not None:
-        return f"!= {quality.mustNotBe}"
-    if quality.mustBeGreaterThan is not None:
-        return f"> {quality.mustBeGreaterThan}"
-    if quality.mustBeGreaterThanOrEqualTo is not None:
-        return f">= {quality.mustBeGreaterThanOrEqualTo}"
-    if quality.mustBeLessThan is not None:
-        return f"< {quality.mustBeLessThan}"
-    if quality.mustBeLessThanOrEqualTo is not None:
-        return f"<= {quality.mustBeLessThanOrEqualTo}"
-    if quality.mustBeBetween is not None:
-        if len(quality.mustBeBetween) != 2:
-            logger.warning(
-                f"Quality check has invalid mustBeBetween, must have exactly 2 integers in an array: {quality.mustBeBetween}"
-            )
-            return None
-        return f"between {quality.mustBeBetween[0]} and {quality.mustBeBetween[1]}"
-    if quality.mustNotBeBetween is not None:
-        if len(quality.mustNotBeBetween) != 2:
-            logger.warning(
-                f"Quality check has invalid mustNotBeBetween, must have exactly 2 integers in an array: {quality.mustNotBeBetween}"
-            )
-            return None
-        return f"not between {quality.mustNotBeBetween[0]} and {quality.mustNotBeBetween[1]}"
-    return None
-
-
-# These are deprecated root-level quality specifications, use the model-level and field-level quality fields instead
-def add_quality_checks(sodacl, data_contract_spec):
-    if data_contract_spec.quality is None:
-        return
-    if data_contract_spec.quality.type is None:
-        return
-    if data_contract_spec.quality.type.lower() != "sodacl":
-        return
-    if isinstance(data_contract_spec.quality.specification, str):
-        quality_specification = yaml.safe_load(data_contract_spec.quality.specification)
-    else:
-        quality_specification = data_contract_spec.quality.specification
-    for key, checks in quality_specification.items():
-        if key in sodacl:
-            for check in checks:
-                sodacl[key].append(check)
-        else:
-            sodacl[key] = checks
+                sodacl_dict[key] = value
+    return yaml.dump(sodacl_dict)

datacontract/export/sql_type_converter.py

@@ -142,11 +142,16 @@ def convert_to_dataframe(field: Field) -> None | str:
     if type.lower() in ["boolean"]:
         return "BOOLEAN"
     if type.lower() in ["object", "record", "struct"]:
-        return "STRUCT"
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_to_dataframe(nested_field)
+            nested_fields.append(f"{nested_field_name}:{nested_field_type}")
+        return f"STRUCT<{','.join(nested_fields)}>"
     if type.lower() in ["bytes"]:
         return "BINARY"
     if type.lower() in ["array"]:
-        return "ARRAY"
+        item_type = convert_to_dataframe(field.items)
+        return f"ARRAY<{item_type}>"
     return None
 
 

datacontract/imports/odcs_importer.py

@@ -41,9 +41,12 @@ def import_odcs(data_contract_specification: DataContractSpecification, source:
         )
 
     if odcs_api_version.startswith("v2."):
-        from datacontract.imports.odcs_v2_importer import import_odcs_v2
-
-        return import_odcs_v2(data_contract_specification, source)
+        raise DataContractException(
+            type="schema",
+            name="Importing ODCS contract",
+            reason=f"Unsupported ODCS API version: {odcs_api_version}",
+            engine="datacontract",
+        )
     elif odcs_api_version.startswith("v3."):
         from datacontract.imports.odcs_v3_importer import import_odcs_v3
 

datacontract/imports/odcs_v3_importer.py

@@ -287,6 +287,8 @@ def import_fields(
 
 
 def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
+    if odcs_type is None:
+        return None
     t = odcs_type.lower()
     if t in DATACONTRACT_TYPES:
         return t