datacontract-cli 0.10.23__py3-none-any.whl → 0.10.25__py3-none-any.whl

datacontract/__init__.py

@@ -0,0 +1,13 @@
+# Configuration so that yaml.safe_dump dumps strings with line breaks with yaml literal |
+import yaml
+
+yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str
+
+
+def repr_str(dumper, data):
+    if "\n" in data:
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+    return dumper.org_represent_str(data)
+
+
+yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)

datacontract/api.py

@@ -162,7 +162,7 @@ async def test(
     server: Annotated[
         str | None,
         Query(
-            example="production",
+            examples=["production"],
             description="The server name to test. Optional, if there is only one server.",
         ),
     ] = None,
@@ -191,7 +191,7 @@ async def lint(
     schema: Annotated[
         str | None,
         Query(
-            example="https://datacontract.com/datacontract.schema.json",
+            examples=["https://datacontract.com/datacontract.schema.json"],
             description="The schema to use for validation. This must be a URL.",
         ),
     ] = None,
@@ -220,7 +220,7 @@ def export(
     server: Annotated[
         str | None,
         Query(
-            example="production",
+            examples=["production"],
             description="The server name to export. Optional, if there is only one server.",
         ),
     ] = None,

datacontract/catalog/catalog.py

@@ -19,7 +19,7 @@ def create_data_contract_html(contracts, file: Path, path: Path, schema: str):
     file_without_suffix = file.with_suffix(".html")
     html_filepath = path / file_without_suffix
     html_filepath.parent.mkdir(parents=True, exist_ok=True)
-    with open(html_filepath, "w") as f:
+    with open(html_filepath, "w", encoding="utf-8") as f:
         f.write(html)
     contracts.append(
         DataContractView(
@@ -42,7 +42,7 @@ class DataContractView:
 
 def create_index_html(contracts, path):
     index_filepath = path / "index.html"
-    with open(index_filepath, "w") as f:
+    with open(index_filepath, "w", encoding="utf-8") as f:
         # Load templates from templates folder
         package_loader = PackageLoader("datacontract", "templates")
         env = Environment(

datacontract/cli.py

@@ -244,7 +244,7 @@ def import_(
     ] = None,
     source: Annotated[
         Optional[str],
-        typer.Option(help="The path to the file or Glue Database that should be imported."),
+        typer.Option(help="The path to the file that should be imported."),
     ] = None,
     dialect: Annotated[
         Optional[str],

datacontract/data_contract.py

@@ -4,6 +4,8 @@ import typing
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
+from duckdb.duckdb import DuckDBPyConnection
+
 from datacontract.breaking.breaking import (
     info_breaking_changes,
     models_breaking_changes,
@@ -22,7 +24,6 @@ from datacontract.lint.linters.description_linter import DescriptionLinter
 from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
 from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
-from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
 from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.model.exceptions import DataContractException
@@ -39,6 +40,7 @@ class DataContract:
         server: str = None,
         publish_url: str = None,
         spark: "SparkSession" = None,
+        duckdb_connection: DuckDBPyConnection = None,
         inline_definitions: bool = True,
         inline_quality: bool = True,
         ssl_verification: bool = True,
@@ -50,11 +52,11 @@ class DataContract:
         self._server = server
         self._publish_url = publish_url
         self._spark = spark
+        self._duckdb_connection = duckdb_connection
         self._inline_definitions = inline_definitions
         self._inline_quality = inline_quality
         self._ssl_verification = ssl_verification
         self.all_linters = {
-            QualityUsesSchemaLinter(),
             FieldPatternLinter(),
             FieldReferenceLinter(),
             NoticePeriodLinter(),
@@ -146,7 +148,7 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
 
-            execute_data_contract_test(data_contract, run, self._server, self._spark)
+            execute_data_contract_test(data_contract, run, self._server, self._spark, self._duckdb_connection)
 
         except DataContractException as e:
             run.checks.append(

datacontract/engines/data_contract_test.py

@@ -1,5 +1,7 @@
 import typing
 
+from duckdb.duckdb import DuckDBPyConnection
+
 from datacontract.engines.data_contract_checks import create_checks
 
 if typing.TYPE_CHECKING:
@@ -10,7 +12,7 @@ from datacontract.engines.datacontract.check_that_datacontract_contains_valid_se
 )
 from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
 from datacontract.engines.soda.check_soda_execute import check_soda_execute
-from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
 from datacontract.model.run import ResultEnum, Run
 
@@ -20,6 +22,7 @@ def execute_data_contract_test(
     run: Run,
     server_name: str = None,
     spark: "SparkSession" = None,
+    duckdb_connection: DuckDBPyConnection = None,
 ):
     if data_contract_specification.models is None or len(data_contract_specification.models) == 0:
         raise DataContractException(
@@ -29,6 +32,12 @@ def execute_data_contract_test(
             reason="Models block is missing. Skip executing tests.",
             engine="datacontract",
         )
+    if (
+        server_name is None
+        and data_contract_specification.servers is not None
+        and len(data_contract_specification.servers) > 0
+    ):
+        server_name = list(data_contract_specification.servers.keys())[0]
     server = get_server(data_contract_specification, server_name)
     run.log_info(f"Running tests for data contract {data_contract_specification.id} with server {server_name}")
     run.dataContractId = data_contract_specification.id
@@ -43,10 +52,10 @@ def execute_data_contract_test(
     # TODO check server credentials are complete for nicer error messages
     if server.format == "json" and server.type != "kafka":
         check_jsonschema(run, data_contract_specification, server)
-    check_soda_execute(run, data_contract_specification, server, spark)
+    check_soda_execute(run, data_contract_specification, server, spark, duckdb_connection)
 
 
-def get_server(data_contract_specification: DataContractSpecification, server_name: str = None):
+def get_server(data_contract_specification: DataContractSpecification, server_name: str = None) -> Server | None:
     """Get the server configuration from the data contract specification.
 
     Args:
@@ -59,7 +68,7 @@ def get_server(data_contract_specification: DataContractSpecification, server_na
 
     check_that_datacontract_contains_valid_server_configuration(data_contract_specification, server_name)
 
-    if server_name:
+    if server_name is not None:
         server = data_contract_specification.servers.get(server_name)
     else:
         server_name = list(data_contract_specification.servers.keys())[0]

datacontract/engines/fastjsonschema/s3/s3_read_files.py

@@ -2,6 +2,7 @@ import logging
 import os
 
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum
 
 
 def yield_s3_files(s3_endpoint_url, s3_location):
@@ -19,9 +20,9 @@ def s3_fs(s3_endpoint_url):
     except ImportError as e:
         raise DataContractException(
             type="schema",
-            result="failed",
+            result=ResultEnum.failed,
             name="s3 extra missing",
-            reason="Install the extra datacontract-cli\[s3] to use s3",
+            reason="Install the extra s3 to use s3",
             engine="datacontract",
             original_exception=e,
         )

datacontract/engines/soda/check_soda_execute.py

@@ -1,6 +1,12 @@
 import logging
+import typing
 import uuid
 
+if typing.TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
+from duckdb.duckdb import DuckDBPyConnection
+
 from datacontract.engines.soda.connections.bigquery import to_bigquery_soda_configuration
 from datacontract.engines.soda.connections.databricks import to_databricks_soda_configuration
 from datacontract.engines.soda.connections.duckdb_connection import get_duckdb_connection
@@ -14,7 +20,13 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
 from datacontract.model.run import Check, Log, ResultEnum, Run
 
 
-def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark):
+def check_soda_execute(
+    run: Run,
+    data_contract: DataContractSpecification,
+    server: Server,
+    spark: "SparkSession" = None,
+    duckdb_connection: DuckDBPyConnection = None,
+):
     from soda.common.config_helper import ConfigHelper
 
     ConfigHelper.get_instance().upsert_value("send_anonymous_usage_stats", False)
@@ -30,7 +42,7 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
     if server.type in ["s3", "gcs", "azure", "local"]:
         if server.format in ["json", "parquet", "csv", "delta"]:
             run.log_info(f"Configuring engine soda-core to connect to {server.type} {server.format} with duckdb")
-            con = get_duckdb_connection(data_contract, server, run)
+            con = get_duckdb_connection(data_contract, server, run, duckdb_connection)
             scan.add_duckdb_connection(duckdb_connection=con, data_source_name=server.type)
             scan.set_data_source_name(server.type)
         else:
@@ -62,7 +74,8 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
             run.log_info("Connecting to databricks via spark")
             scan.add_spark_session(spark, data_source_name=server.type)
             scan.set_data_source_name(server.type)
-            spark.sql(f"USE {server.catalog}.{server.schema_}")
+            database_name = ".".join(filter(None, [server.catalog, server.schema_]))
+            spark.sql(f"USE {database_name}")
         else:
             run.log_info("Connecting to databricks directly")
             soda_configuration_str = to_databricks_soda_configuration(server)

datacontract/engines/soda/connections/duckdb_connection.py

@@ -1,14 +1,24 @@
 import os
-from typing import Any
+from typing import Any, Dict
 
 import duckdb
 
-from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
+from datacontract.export.duckdb_type_converter import convert_to_duckdb_csv_type, convert_to_duckdb_json_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server
 from datacontract.model.run import Run
 
 
-def get_duckdb_connection(data_contract, server, run: Run):
-    con = duckdb.connect(database=":memory:")
+def get_duckdb_connection(
+    data_contract: DataContractSpecification,
+    server: Server,
+    run: Run,
+    duckdb_connection: duckdb.DuckDBPyConnection | None = None,
+) -> duckdb.DuckDBPyConnection:
+    if duckdb_connection is None:
+        con = duckdb.connect(database=":memory:")
+    else:
+        con = duckdb_connection
+
     path: str = ""
     if server.type == "local":
         path = server.path
@@ -33,9 +43,16 @@ def get_duckdb_connection(data_contract, server, run: Run):
                 json_format = "newline_delimited"
             elif server.delimiter == "array":
                 json_format = "array"
-            con.sql(f"""
+            columns = to_json_types(model)
+            if columns is None:
+                con.sql(f"""
                         CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{json_format}', hive_partitioning=1);
                         """)
+            else:
+                con.sql(
+                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{json_format}', columns={columns}, hive_partitioning=1);"""
+                )
+                add_nested_views(con, model_name, model.fields)
         elif server.format == "parquet":
             con.sql(f"""
                         CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1);
@@ -67,6 +84,45 @@ def to_csv_types(model) -> dict[Any, str | None] | None:
     return columns
 
 
+def to_json_types(model: Model) -> dict[Any, str | None] | None:
+    if model is None:
+        return None
+    columns = {}
+    for field_name, field in model.fields.items():
+        columns[field_name] = convert_to_duckdb_json_type(field)
+    return columns
+
+
+def add_nested_views(con: duckdb.DuckDBPyConnection, model_name: str, fields: Dict[str, Field] | None):
+    model_name = model_name.strip('"')
+    if fields is None:
+        return
+    for field_name, field in fields.items():
+        if field.type is None or field.type.lower() not in ["array", "object"]:
+            continue
+        field_type = field.type.lower()
+        if field_type == "array" and field.items is None:
+            continue
+        elif field_type == "object" and field.fields is None:
+            continue
+
+        nested_model_name = f"{model_name}__{field_name}"
+        max_depth = 2 if field_type == "array" else 1
+
+        ## if parent field is not required, the nested objects may respolve
+        ## to a row of NULLs -- but if the objects themselves have required
+        ## fields, this will fail the check.
+        where = "" if field.required else f" WHERE {field_name} IS NOT NULL"
+        con.sql(f"""
+            CREATE VIEW IF NOT EXISTS "{nested_model_name}" AS
+            SELECT unnest({field_name}, max_depth := {max_depth}) as {field_name} FROM "{model_name}" {where}
+            """)
+        if field_type == "array":
+            add_nested_views(con, nested_model_name, field.items.fields)
+        elif field_type == "object":
+            add_nested_views(con, nested_model_name, field.fields)
+
+
 def setup_s3_connection(con, server):
     s3_region = os.getenv("DATACONTRACT_S3_REGION")
     s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")

datacontract/engines/soda/connections/kafka.py

@@ -6,6 +6,7 @@ import tempfile
 from datacontract.export.avro_converter import to_avro_schema_json
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Server
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum
 
 
 def create_spark_session():
@@ -16,7 +17,7 @@ def create_spark_session():
     except ImportError as e:
         raise DataContractException(
             type="schema",
-            result="failed",
+            result=ResultEnum.failed,
             name="pyspark is missing",
             reason="Install the extra datacontract-cli[kafka] to use kafka",
             engine="datacontract",
@@ -33,7 +34,7 @@ def create_spark_session():
         .config("spark.ui.enabled", "false")
         .config(
             "spark.jars.packages",
-            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.2,org.apache.spark:spark-avro_2.12:3.5.2",
+            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5,org.apache.spark:spark-avro_2.12:3.5.5",
         )
         .getOrCreate()
     )

datacontract/export/avro_converter.py

@@ -91,7 +91,9 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         if field.precision is not None:
             typeVal["precision"] = field.precision
         return typeVal
-    elif field.type in ["float", "double"]:
+    elif field.type in ["float"]:
+        return "float"
+    elif field.type in ["double"]:
         return "double"
     elif field.type in ["integer", "int"]:
         return "int"
@@ -107,6 +109,11 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         return {"type": "int", "logicalType": "date"}
     elif field.type in ["time"]:
         return "long"
+    elif field.type in ["map"]:
+        if field.config is not None and "values" in field.config:
+            return {"type": "map", "values": field.config["values"]}
+        else:
+            return "bytes"
     elif field.type in ["object", "record", "struct"]:
         if field.config is not None and "namespace" in field.config:
             return to_avro_record(field_name, field.fields, field.description, field.config["namespace"])

datacontract/export/bigquery_converter.py

@@ -103,7 +103,7 @@ def map_type_to_bigquery(field: Field) -> str:
     elif field_type.lower() == "date":
         return "DATE"
     elif field_type.lower() == "timestamp_ntz":
-        return "TIME"
+        return "DATETIME"
     elif field_type.lower() in ["number", "decimal", "numeric"]:
         return "NUMERIC"
     elif field_type.lower() == "double":

datacontract/export/duckdb_type_converter.py

@@ -0,0 +1,57 @@
+from typing import Dict
+
+from datacontract.model.data_contract_specification import Field
+
+
+# https://duckdb.org/docs/data/csv/overview.html
+# ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR']
+def convert_to_duckdb_csv_type(field) -> None | str:
+    datacontract_type = field.type
+    if datacontract_type is None:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["string", "varchar", "text"]:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if datacontract_type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP"
+    if datacontract_type.lower() in ["date"]:
+        return "DATE"
+    if datacontract_type.lower() in ["time"]:
+        return "TIME"
+    if datacontract_type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "VARCHAR"
+    if datacontract_type.lower() in ["float", "double"]:
+        return "DOUBLE"
+    if datacontract_type.lower() in ["integer", "int", "long", "bigint"]:
+        return "BIGINT"
+    if datacontract_type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if datacontract_type.lower() in ["object", "record", "struct"]:
+        # not supported in CSV
+        return "VARCHAR"
+    if datacontract_type.lower() in ["bytes"]:
+        # not supported in CSV
+        return "VARCHAR"
+    if datacontract_type.lower() in ["array"]:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["null"]:
+        return "SQLNULL"
+    return "VARCHAR"
+
+
+def convert_to_duckdb_json_type(field: Field) -> None | str:
+    datacontract_type = field.type
+    if datacontract_type is None:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["array"]:
+        return convert_to_duckdb_json_type(field.items) + "[]"  # type: ignore
+    if datacontract_type.lower() in ["object", "record", "struct"]:
+        return convert_to_duckdb_object(field.fields)
+    return convert_to_duckdb_csv_type(field)
+
+
+def convert_to_duckdb_object(fields: Dict[str, Field]):
+    columns = [f'"{x[0]}" {convert_to_duckdb_json_type(x[1])}' for x in fields.items()]
+    return f"STRUCT({', '.join(columns)})"

datacontract/export/great_expectations_converter.py

@@ -19,6 +19,7 @@ from datacontract.export.spark_converter import to_spark_data_type
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
+    DeprecatedQuality,
     Field,
     Quality,
 )
@@ -91,8 +92,14 @@ def to_great_expectations(
             model_key=model_key, contract_version=data_contract_spec.info.version
         )
     model_value = data_contract_spec.models.get(model_key)
-    quality_checks = get_quality_checks(data_contract_spec.quality)
+
+    # Support for Deprecated Quality
+    quality_checks = get_deprecated_quality_checks(data_contract_spec.quality)
+
+    expectations.extend(get_quality_checks(model_value.quality))
+
     expectations.extend(model_to_expectations(model_value.fields, engine, sql_server_type))
+
     expectations.extend(checks_to_expectations(quality_checks, model_key))
     model_expectation_suite = to_suite(expectations, expectation_suite_name)
 
@@ -135,6 +142,7 @@ def model_to_expectations(fields: Dict[str, Field], engine: str | None, sql_serv
     add_column_order_exp(fields, expectations)
     for field_name, field in fields.items():
         add_field_expectations(field_name, field, expectations, engine, sql_server_type)
+        expectations.extend(get_quality_checks(field.quality, field_name))
     return expectations
 
 
@@ -173,6 +181,8 @@ def add_field_expectations(
         expectations.append(to_column_length_exp(field_name, field.minLength, field.maxLength))
     if field.minimum is not None or field.maximum is not None:
         expectations.append(to_column_min_max_exp(field_name, field.minimum, field.maximum))
+    if field.enum is not None and len(field.enum) != 0:
+        expectations.append(to_column_enum_exp(field_name, field.enum))
 
     return expectations
 
@@ -266,7 +276,24 @@ def to_column_min_max_exp(field_name, minimum, maximum) -> Dict[str, Any]:
     }
 
 
-def get_quality_checks(quality: Quality) -> Dict[str, Any]:
+def to_column_enum_exp(field_name, enum_list: List[str]) -> Dict[str, Any]:
+    """Creates a expect_column_values_to_be_in_set expectation.
+
+    Args:
+        field_name (str): The name of the field.
+        enum_list (Set[str]): enum list of value.
+
+    Returns:
+        Dict[str, Any]: Column value in set expectation.
+    """
+    return {
+        "expectation_type": "expect_column_values_to_be_in_set",
+        "kwargs": {"column": field_name, "value_set": enum_list},
+        "meta": {},
+    }
+
+
+def get_deprecated_quality_checks(quality: DeprecatedQuality) -> Dict[str, Any]:
     """Retrieves quality checks defined in a data contract.
 
     Args:
@@ -288,6 +315,26 @@ def get_quality_checks(quality: Quality) -> Dict[str, Any]:
     return quality_specification
 
 
+def get_quality_checks(qualities: List[Quality], field_name: str | None = None) -> List[Dict[str, Any]]:
+    """Retrieves quality checks defined in a data contract.
+
+    Args:
+        qualities (List[Quality]): List of quality object from the model specification.
+        field_name (str | None): field name if the quality list is attached to a specific field
+
+    Returns:
+        Dict[str, Any]: Dictionary of quality checks.
+    """
+    quality_specification = []
+    for quality in qualities:
+        if quality is not None and quality.engine is not None and quality.engine.lower() == "great-expectations":
+            ge_expectation = quality.implementation
+            if field_name is not None:
+                ge_expectation["column"] = field_name
+            quality_specification.append(ge_expectation)
+    return quality_specification
+
+
 def checks_to_expectations(quality_checks: Dict[str, Any], model_key: str) -> List[Dict[str, Any]]:
     """Converts quality checks to a list of expectations.