datacontract-cli 0.10.21__py3-none-any.whl → 0.10.23__py3-none-any.whl

datacontract/engines/data_contract_test.py

@@ -0,0 +1,67 @@
+import typing
+
+from datacontract.engines.data_contract_checks import create_checks
+
+if typing.TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
+from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
+    check_that_datacontract_contains_valid_server_configuration,
+)
+from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
+from datacontract.engines.soda.check_soda_execute import check_soda_execute
+from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum, Run
+
+
+def execute_data_contract_test(
+    data_contract_specification: DataContractSpecification,
+    run: Run,
+    server_name: str = None,
+    spark: "SparkSession" = None,
+):
+    if data_contract_specification.models is None or len(data_contract_specification.models) == 0:
+        raise DataContractException(
+            type="lint",
+            name="Check that data contract contains models",
+            result=ResultEnum.warning,
+            reason="Models block is missing. Skip executing tests.",
+            engine="datacontract",
+        )
+    server = get_server(data_contract_specification, server_name)
+    run.log_info(f"Running tests for data contract {data_contract_specification.id} with server {server_name}")
+    run.dataContractId = data_contract_specification.id
+    run.dataContractVersion = data_contract_specification.info.version
+    run.dataProductId = server.dataProductId
+    run.outputPortId = server.outputPortId
+    run.server = server_name
+
+    run.checks.extend(create_checks(data_contract_specification, server))
+
+    # TODO check server is supported type for nicer error messages
+    # TODO check server credentials are complete for nicer error messages
+    if server.format == "json" and server.type != "kafka":
+        check_jsonschema(run, data_contract_specification, server)
+    check_soda_execute(run, data_contract_specification, server, spark)
+
+
+def get_server(data_contract_specification: DataContractSpecification, server_name: str = None):
+    """Get the server configuration from the data contract specification.
+
+    Args:
+        data_contract_specification: The data contract specification
+        server_name: Optional name of the server to use. If not provided, uses the first server.
+
+    Returns:
+        The selected server configuration
+    """
+
+    check_that_datacontract_contains_valid_server_configuration(data_contract_specification, server_name)
+
+    if server_name:
+        server = data_contract_specification.servers.get(server_name)
+    else:
+        server_name = list(data_contract_specification.servers.keys())[0]
+        server = data_contract_specification.servers.get(server_name)
+    return server

datacontract/engines/datacontract/check_that_datacontract_contains_valid_servers_configuration.py

@@ -1,12 +1,11 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Run
 
 
 def check_that_datacontract_contains_valid_server_configuration(
-    run: Run, data_contract: DataContractSpecification, server_name: str
+    data_contract: DataContractSpecification, server_name: str | None
 ):
-    if data_contract.servers is None:
+    if data_contract.servers is None or len(data_contract.servers) == 0:
         raise DataContractException(
             type="lint",
             name="Check that data contract contains valid server configuration",

datacontract/engines/soda/check_soda_execute.py

@@ -1,8 +1,9 @@
 import logging
+import uuid
 
 from datacontract.engines.soda.connections.bigquery import to_bigquery_soda_configuration
 from datacontract.engines.soda.connections.databricks import to_databricks_soda_configuration
-from datacontract.engines.soda.connections.duckdb import get_duckdb_connection
+from datacontract.engines.soda.connections.duckdb_connection import get_duckdb_connection
 from datacontract.engines.soda.connections.kafka import create_spark_session, read_kafka_topic
 from datacontract.engines.soda.connections.postgres import to_postgres_soda_configuration
 from datacontract.engines.soda.connections.snowflake import to_snowflake_soda_configuration
@@ -13,7 +14,7 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
 from datacontract.model.run import Check, Log, ResultEnum, Run
 
 
-def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
+def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark):
     from soda.common.config_helper import ConfigHelper
 
     ConfigHelper.get_instance().upsert_value("send_anonymous_usage_stats", False)
@@ -80,8 +81,8 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
             scan.set_data_source_name("datacontract-cli")
     elif server.type == "kafka":
         if spark is None:
-            spark = create_spark_session(tmp_dir)
-        read_kafka_topic(spark, data_contract, server, tmp_dir)
+            spark = create_spark_session()
+        read_kafka_topic(spark, data_contract, server)
         scan.add_spark_session(spark, data_source_name=server.type)
         scan.set_data_source_name(server.type)
     elif server.type == "sqlserver":
@@ -106,37 +107,34 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         run.log_warn(f"Server type {server.type} not yet supported by datacontract CLI")
         return
 
-    # Don't check types for json format, as they are checked with json schema
-    # Don't check types for avro format, as they are checked with avro schema
-    # Don't check types for csv format, as they are hard to detect
-    server_type = server.type
-    check_types = server.format != "json" and server.format != "csv" and server.format != "avro"
-
-    sodacl_yaml_str = to_sodacl_yaml(data_contract, server_type, check_types)
+    sodacl_yaml_str = to_sodacl_yaml(run)
     # print("sodacl_yaml_str:\n" + sodacl_yaml_str)
     scan.add_sodacl_yaml_str(sodacl_yaml_str)
 
     # Execute the scan
-    logging.info("Starting soda scan")
+    logging.info("Starting soda scan with checks:\n" + sodacl_yaml_str)
     scan.execute()
     logging.info("Finished soda scan")
 
     # pprint.PrettyPrinter(indent=2).pprint(scan.build_scan_results())
 
     scan_results = scan.get_scan_results()
-    for c in scan_results.get("checks"):
-        check = Check(
-            type="schema",
-            result=to_result(c),
-            reason=", ".join(c.get("outcomeReasons")),
-            name=c.get("name"),
-            model=c.get("table"),
-            field=c.get("column"),
-            engine="soda-core",
-            diagnostics=c.get("diagnostics"),
-        )
-        update_reason(check, c)
-        run.checks.append(check)
+    for scan_result in scan_results.get("checks"):
+        name = scan_result.get("name")
+        check = get_check(run, scan_result)
+        if check is None:
+            check = Check(
+                id=str(uuid.uuid4()),
+                category="custom",
+                type="custom",
+                name=name,
+                engine="soda-core",
+            )
+            run.checks.append(check)
+        check.result = to_result(scan_result)
+        check.reason = ", ".join(scan_result.get("outcomeReasons"))
+        check.diagnostics = scan_result.get("diagnostics")
+        update_reason(check, scan_result)
 
     for log in scan_results.get("logs"):
         run.logs.append(
@@ -152,8 +150,8 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         run.checks.append(
             Check(
                 type="general",
-                name="Execute quality checks",
-                result="warning",
+                name="Data Contract Tests",
+                result=ResultEnum.warning,
                 reason="Engine soda-core has errors. See the logs for details.",
                 engine="soda-core",
             )
@@ -161,14 +159,22 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
         return
 
 
-def to_result(c) -> str:
+def get_check(run, scan_result) -> Check | None:
+    check_by_name = next((c for c in run.checks if c.key == scan_result.get("name")), None)
+    if check_by_name is not None:
+        return check_by_name
+
+    return None
+
+
+def to_result(c) -> ResultEnum:
     soda_outcome = c.get("outcome")
     if soda_outcome == "pass":
-        return "passed"
+        return ResultEnum.passed
     elif soda_outcome == "fail":
-        return "failed"
+        return ResultEnum.failed
     else:
-        return soda_outcome
+        return ResultEnum.unknown
 
 
 def update_reason(check, c):

datacontract/engines/soda/connections/{duckdb.py → duckdb_connection.py}

@@ -1,4 +1,5 @@
 import os
+from typing import Any
 
 import duckdb
 
@@ -27,13 +28,13 @@ def get_duckdb_connection(data_contract, server, run: Run):
         run.log_info(f"Creating table {model_name} for {model_path}")
 
         if server.format == "json":
-            format = "auto"
+            json_format = "auto"
             if server.delimiter == "new_line":
-                format = "newline_delimited"
+                json_format = "newline_delimited"
             elif server.delimiter == "array":
-                format = "array"
+                json_format = "array"
             con.sql(f"""
-                        CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{format}', hive_partitioning=1);
+                        CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{json_format}', hive_partitioning=1);
                         """)
         elif server.format == "parquet":
             con.sql(f"""
@@ -56,7 +57,7 @@ def get_duckdb_connection(data_contract, server, run: Run):
     return con
 
 
-def to_csv_types(model) -> dict:
+def to_csv_types(model) -> dict[Any, str | None] | None:
     if model is None:
         return None
     columns = {}

datacontract/engines/soda/connections/kafka.py

@@ -1,12 +1,14 @@
+import atexit
 import logging
 import os
+import tempfile
 
 from datacontract.export.avro_converter import to_avro_schema_json
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Server
 from datacontract.model.exceptions import DataContractException
 
 
-def create_spark_session(tmp_dir: str):
+def create_spark_session():
     """Create and configure a Spark session."""
 
     try:
@@ -21,6 +23,9 @@ def create_spark_session(tmp_dir: str):
             original_exception=e,
         )
 
+    tmp_dir = tempfile.TemporaryDirectory(prefix="datacontract-cli-spark")
+    atexit.register(tmp_dir.cleanup)
+
     spark = (
         SparkSession.builder.appName("datacontract")
         .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse")
@@ -37,7 +42,7 @@ def create_spark_session(tmp_dir: str):
     return spark
 
 
-def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Server, tmp_dir):
+def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Server):
     """Read and process data from a Kafka topic based on the server configuration."""
 
     logging.info("Reading data from Kafka server %s topic %s", server.host, server.topic)
@@ -62,7 +67,7 @@ def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Se
                 type="test",
                 name="Configuring Kafka checks",
                 result="warning",
-                reason=f"Kafka format '{server.format}' is not supported. " f"Skip executing tests.",
+                reason=f"Kafka format '{server.format}' is not supported. Skip executing tests.",
                 engine="datacontract",
             )
 

datacontract/export/avro_converter.py

@@ -108,6 +108,8 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
     elif field.type in ["time"]:
         return "long"
     elif field.type in ["object", "record", "struct"]:
+        if field.config is not None and "namespace" in field.config:
+            return to_avro_record(field_name, field.fields, field.description, field.config["namespace"])
         return to_avro_record(field_name, field.fields, field.description, None)
     elif field.type in ["binary"]:
         return "bytes"

datacontract/export/dbt_converter.py

@@ -9,7 +9,7 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
 
 class DbtExporter(Exporter):
     def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
-        return to_dbt_models_yaml(data_contract)
+        return to_dbt_models_yaml(data_contract, server)
 
 
 class DbtSourceExporter(Exporter):
@@ -27,15 +27,16 @@ class DbtStageExporter(Exporter):
         )
 
 
-def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
+def to_dbt_models_yaml(data_contract_spec: DataContractSpecification, server: str = None):
     dbt = {
         "version": 2,
         "models": [],
     }
+
     for model_key, model_value in data_contract_spec.models.items():
-        dbt_model = _to_dbt_model(model_key, model_value, data_contract_spec)
+        dbt_model = _to_dbt_model(model_key, model_value, data_contract_spec, adapter_type=server)
         dbt["models"].append(dbt_model)
-    return yaml.dump(dbt, indent=2, sort_keys=False, allow_unicode=True)
+    return yaml.safe_dump(dbt, indent=2, sort_keys=False, allow_unicode=True)
 
 
 def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name: str, model_value: Model) -> str:
@@ -60,7 +61,7 @@ def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: s
     if data_contract_spec.info.owner is not None:
         source["meta"] = {"owner": data_contract_spec.info.owner}
     if data_contract_spec.info.description is not None:
-        source["description"] = data_contract_spec.info.description
+        source["description"] = data_contract_spec.info.description.strip().replace("\n", " ")
     found_server = data_contract_spec.servers.get(server)
     adapter_type = None
     if found_server is not None:
@@ -87,14 +88,16 @@ def _to_dbt_source_table(
     }
 
     if model_value.description is not None:
-        dbt_model["description"] = model_value.description
+        dbt_model["description"] = model_value.description.strip().replace("\n", " ")
     columns = _to_columns(data_contract_spec, model_value.fields, False, adapter_type)
     if columns:
         dbt_model["columns"] = columns
     return dbt_model
 
 
-def _to_dbt_model(model_key, model_value: Model, data_contract_spec: DataContractSpecification) -> dict:
+def _to_dbt_model(
+    model_key, model_value: Model, data_contract_spec: DataContractSpecification, adapter_type: Optional[str]
+) -> dict:
     dbt_model = {
         "name": model_key,
     }
@@ -108,8 +111,8 @@ def _to_dbt_model(model_key, model_value: Model, data_contract_spec: DataContrac
     if _supports_constraints(model_type):
         dbt_model["config"]["contract"] = {"enforced": True}
     if model_value.description is not None:
-        dbt_model["description"] = model_value.description
-    columns = _to_columns(data_contract_spec, model_value.fields, _supports_constraints(model_type), None)
+        dbt_model["description"] = model_value.description.strip().replace("\n", " ")
+    columns = _to_columns(data_contract_spec, model_value.fields, _supports_constraints(model_type), adapter_type)
     if columns:
         dbt_model["columns"] = columns
     return dbt_model
@@ -171,7 +174,7 @@ def _to_column(
             {"dbt_expectations.dbt_expectations.expect_column_values_to_be_of_type": {"column_type": dbt_type}}
         )
     if field.description is not None:
-        column["description"] = field.description
+        column["description"] = field.description.strip().replace("\n", " ")
     if field.required:
         if supports_constraints:
             column.setdefault("constraints", []).append({"type": "not_null"})

datacontract/export/exporter.py

@@ -25,8 +25,6 @@ class ExportFormat(str, Enum):
     dbt_sources = "dbt-sources"
     dbt_staging_sql = "dbt-staging-sql"
     odcs = "odcs"
-    odcs_v2 = "odcs_v2"
-    odcs_v3 = "odcs_v3"
     rdf = "rdf"
     avro = "avro"
     protobuf = "protobuf"

datacontract/export/exporter_factory.py

@@ -107,18 +107,6 @@ exporter_factory.register_lazy_exporter(
     class_name="JsonSchemaExporter",
 )
 
-exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v2,
-    module_path="datacontract.export.odcs_v2_exporter",
-    class_name="OdcsV2Exporter",
-)
-
-exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs_v3,
-    module_path="datacontract.export.odcs_v3_exporter",
-    class_name="OdcsV3Exporter",
-)
-
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.odcs,
     module_path="datacontract.export.odcs_v3_exporter",

datacontract/export/odcs_v3_exporter.py

@@ -19,7 +19,7 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
         "name": data_contract_spec.info.title,
         "version": data_contract_spec.info.version,
         "domain": data_contract_spec.info.owner,
-        "status": data_contract_spec.info.status,
+        "status": to_status(data_contract_spec.info.status),
     }
 
     if data_contract_spec.terms is not None:
@@ -217,9 +217,9 @@ def to_property(field_name: str, field: Field) -> dict:
     if field.description is not None:
         property["description"] = field.description
     if field.required is not None:
-        property["isNullable"] = not field.required
+        property["nullable"] = not field.required
     if field.unique is not None:
-        property["isUnique"] = field.unique
+        property["unique"] = field.unique
     if field.classification is not None:
         property["classification"] = field.classification
     if field.examples is not None:
@@ -312,3 +312,22 @@ def to_odcs_quality(quality):
     if quality.implementation is not None:
         quality_dict["implementation"] = quality.implementation
     return quality_dict
+
+
+def to_status(status):
+    """Convert the data contract status to ODCS v3 format."""
+    if status is None:
+        return "draft"  # Default to draft if no status is provided
+
+    # Valid status values according to ODCS v3.0.1 spec
+    valid_statuses = ["proposed", "draft", "active", "deprecated", "retired"]
+
+    # Convert to lowercase for comparison
+    status_lower = status.lower()
+
+    # If status is already valid, return it as is
+    if status_lower in valid_statuses:
+        return status_lower
+
+    # Default to "draft" for any non-standard status
+    return "draft"