datacontract-cli 0.10.23__py3-none-any.whl → 0.10.24__py3-none-any.whl

datacontract/__init__.py

@@ -0,0 +1,13 @@
+# Configuration so that yaml.safe_dump dumps strings with line breaks with yaml literal |
+import yaml
+
+yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str
+
+
+def repr_str(dumper, data):
+    if "\n" in data:
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+    return dumper.org_represent_str(data)
+
+
+yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)

datacontract/catalog/catalog.py

@@ -19,7 +19,7 @@ def create_data_contract_html(contracts, file: Path, path: Path, schema: str):
     file_without_suffix = file.with_suffix(".html")
     html_filepath = path / file_without_suffix
     html_filepath.parent.mkdir(parents=True, exist_ok=True)
-    with open(html_filepath, "w") as f:
+    with open(html_filepath, "w", encoding="utf-8") as f:
         f.write(html)
     contracts.append(
         DataContractView(
@@ -42,7 +42,7 @@ class DataContractView:
 
 def create_index_html(contracts, path):
     index_filepath = path / "index.html"
-    with open(index_filepath, "w") as f:
+    with open(index_filepath, "w", encoding="utf-8") as f:
         # Load templates from templates folder
         package_loader = PackageLoader("datacontract", "templates")
         env = Environment(

datacontract/data_contract.py

@@ -4,6 +4,8 @@ import typing
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
+from duckdb.duckdb import DuckDBPyConnection
+
 from datacontract.breaking.breaking import (
     info_breaking_changes,
     models_breaking_changes,
@@ -22,7 +24,6 @@ from datacontract.lint.linters.description_linter import DescriptionLinter
 from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
 from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
-from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
 from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.model.exceptions import DataContractException
@@ -39,6 +40,7 @@ class DataContract:
         server: str = None,
         publish_url: str = None,
         spark: "SparkSession" = None,
+        duckdb_connection: DuckDBPyConnection = None,
         inline_definitions: bool = True,
         inline_quality: bool = True,
         ssl_verification: bool = True,
@@ -50,11 +52,11 @@ class DataContract:
         self._server = server
         self._publish_url = publish_url
         self._spark = spark
+        self._duckdb_connection = duckdb_connection
         self._inline_definitions = inline_definitions
         self._inline_quality = inline_quality
         self._ssl_verification = ssl_verification
         self.all_linters = {
-            QualityUsesSchemaLinter(),
             FieldPatternLinter(),
             FieldReferenceLinter(),
             NoticePeriodLinter(),
@@ -146,7 +148,7 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
 
-            execute_data_contract_test(data_contract, run, self._server, self._spark)
+            execute_data_contract_test(data_contract, run, self._server, self._spark, self._duckdb_connection)
 
         except DataContractException as e:
             run.checks.append(

datacontract/engines/data_contract_test.py

@@ -1,5 +1,7 @@
 import typing
 
+from duckdb.duckdb import DuckDBPyConnection
+
 from datacontract.engines.data_contract_checks import create_checks
 
 if typing.TYPE_CHECKING:
@@ -10,7 +12,7 @@ from datacontract.engines.datacontract.check_that_datacontract_contains_valid_se
 )
 from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
 from datacontract.engines.soda.check_soda_execute import check_soda_execute
-from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
 from datacontract.model.run import ResultEnum, Run
 
@@ -20,6 +22,7 @@ def execute_data_contract_test(
     run: Run,
     server_name: str = None,
     spark: "SparkSession" = None,
+    duckdb_connection: DuckDBPyConnection = None,
 ):
     if data_contract_specification.models is None or len(data_contract_specification.models) == 0:
         raise DataContractException(
@@ -29,6 +32,12 @@ def execute_data_contract_test(
             reason="Models block is missing. Skip executing tests.",
             engine="datacontract",
         )
+    if (
+        server_name is None
+        and data_contract_specification.servers is not None
+        and len(data_contract_specification.servers) > 0
+    ):
+        server_name = list(data_contract_specification.servers.keys())[0]
     server = get_server(data_contract_specification, server_name)
     run.log_info(f"Running tests for data contract {data_contract_specification.id} with server {server_name}")
     run.dataContractId = data_contract_specification.id
@@ -43,10 +52,10 @@ def execute_data_contract_test(
     # TODO check server credentials are complete for nicer error messages
     if server.format == "json" and server.type != "kafka":
         check_jsonschema(run, data_contract_specification, server)
-    check_soda_execute(run, data_contract_specification, server, spark)
+    check_soda_execute(run, data_contract_specification, server, spark, duckdb_connection)
 
 
-def get_server(data_contract_specification: DataContractSpecification, server_name: str = None):
+def get_server(data_contract_specification: DataContractSpecification, server_name: str = None) -> Server | None:
     """Get the server configuration from the data contract specification.
 
     Args:
@@ -59,7 +68,7 @@ def get_server(data_contract_specification: DataContractSpecification, server_na
 
     check_that_datacontract_contains_valid_server_configuration(data_contract_specification, server_name)
 
-    if server_name:
+    if server_name is not None:
         server = data_contract_specification.servers.get(server_name)
     else:
         server_name = list(data_contract_specification.servers.keys())[0]

datacontract/engines/fastjsonschema/s3/s3_read_files.py

@@ -2,6 +2,7 @@ import logging
 import os
 
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum
 
 
 def yield_s3_files(s3_endpoint_url, s3_location):
@@ -19,9 +20,9 @@ def s3_fs(s3_endpoint_url):
     except ImportError as e:
         raise DataContractException(
             type="schema",
-            result="failed",
+            result=ResultEnum.failed,
             name="s3 extra missing",
-            reason="Install the extra datacontract-cli\[s3] to use s3",
+            reason="Install the extra s3 to use s3",
             engine="datacontract",
             original_exception=e,
         )

datacontract/engines/soda/check_soda_execute.py

@@ -1,6 +1,12 @@
 import logging
+import typing
 import uuid
 
+if typing.TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
+from duckdb.duckdb import DuckDBPyConnection
+
 from datacontract.engines.soda.connections.bigquery import to_bigquery_soda_configuration
 from datacontract.engines.soda.connections.databricks import to_databricks_soda_configuration
 from datacontract.engines.soda.connections.duckdb_connection import get_duckdb_connection
@@ -14,7 +20,13 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
 from datacontract.model.run import Check, Log, ResultEnum, Run
 
 
-def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark):
+def check_soda_execute(
+    run: Run,
+    data_contract: DataContractSpecification,
+    server: Server,
+    spark: "SparkSession" = None,
+    duckdb_connection: DuckDBPyConnection = None,
+):
     from soda.common.config_helper import ConfigHelper
 
     ConfigHelper.get_instance().upsert_value("send_anonymous_usage_stats", False)
@@ -30,7 +42,7 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
     if server.type in ["s3", "gcs", "azure", "local"]:
         if server.format in ["json", "parquet", "csv", "delta"]:
             run.log_info(f"Configuring engine soda-core to connect to {server.type} {server.format} with duckdb")
-            con = get_duckdb_connection(data_contract, server, run)
+            con = get_duckdb_connection(data_contract, server, run, duckdb_connection)
             scan.add_duckdb_connection(duckdb_connection=con, data_source_name=server.type)
             scan.set_data_source_name(server.type)
         else:
@@ -62,7 +74,8 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
             run.log_info("Connecting to databricks via spark")
             scan.add_spark_session(spark, data_source_name=server.type)
             scan.set_data_source_name(server.type)
-            spark.sql(f"USE {server.catalog}.{server.schema_}")
+            database_name = ".".join(filter(None, [server.catalog, server.schema_]))
+            spark.sql(f"USE {database_name}")
         else:
             run.log_info("Connecting to databricks directly")
             soda_configuration_str = to_databricks_soda_configuration(server)

datacontract/engines/soda/connections/duckdb_connection.py

@@ -1,14 +1,24 @@
 import os
-from typing import Any
+from typing import Any, Dict
 
 import duckdb
 
-from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
+from datacontract.export.duckdb_type_converter import convert_to_duckdb_csv_type, convert_to_duckdb_json_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server
 from datacontract.model.run import Run
 
 
-def get_duckdb_connection(data_contract, server, run: Run):
-    con = duckdb.connect(database=":memory:")
+def get_duckdb_connection(
+    data_contract: DataContractSpecification,
+    server: Server,
+    run: Run,
+    duckdb_connection: duckdb.DuckDBPyConnection | None = None,
+) -> duckdb.DuckDBPyConnection:
+    if duckdb_connection is None:
+        con = duckdb.connect(database=":memory:")
+    else:
+        con = duckdb_connection
+
     path: str = ""
     if server.type == "local":
         path = server.path
@@ -33,9 +43,16 @@ def get_duckdb_connection(data_contract, server, run: Run):
                 json_format = "newline_delimited"
             elif server.delimiter == "array":
                 json_format = "array"
-            con.sql(f"""
+            columns = to_json_types(model)
+            if columns is None:
+                con.sql(f"""
                         CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{json_format}', hive_partitioning=1);
                         """)
+            else:
+                con.sql(
+                    f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{json_format}', columns={columns}, hive_partitioning=1);"""
+                )
+                add_nested_views(con, model_name, model.fields)
         elif server.format == "parquet":
             con.sql(f"""
                         CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1);
@@ -67,6 +84,45 @@ def to_csv_types(model) -> dict[Any, str | None] | None:
     return columns
 
 
+def to_json_types(model: Model) -> dict[Any, str | None] | None:
+    if model is None:
+        return None
+    columns = {}
+    for field_name, field in model.fields.items():
+        columns[field_name] = convert_to_duckdb_json_type(field)
+    return columns
+
+
+def add_nested_views(con: duckdb.DuckDBPyConnection, model_name: str, fields: Dict[str, Field] | None):
+    model_name = model_name.strip('"')
+    if fields is None:
+        return
+    for field_name, field in fields.items():
+        if field.type is None or field.type.lower() not in ["array", "object"]:
+            continue
+        field_type = field.type.lower()
+        if field_type == "array" and field.items is None:
+            continue
+        elif field_type == "object" and field.fields is None:
+            continue
+
+        nested_model_name = f"{model_name}__{field_name}"
+        max_depth = 2 if field_type == "array" else 1
+
+        ## if parent field is not required, the nested objects may respolve
+        ## to a row of NULLs -- but if the objects themselves have required
+        ## fields, this will fail the check.
+        where = "" if field.required else f" WHERE {field_name} IS NOT NULL"
+        con.sql(f"""
+            CREATE VIEW IF NOT EXISTS "{nested_model_name}" AS
+            SELECT unnest({field_name}, max_depth := {max_depth}) as {field_name} FROM "{model_name}" {where}
+            """)
+        if field_type == "array":
+            add_nested_views(con, nested_model_name, field.items.fields)
+        elif field_type == "object":
+            add_nested_views(con, nested_model_name, field.fields)
+
+
 def setup_s3_connection(con, server):
     s3_region = os.getenv("DATACONTRACT_S3_REGION")
     s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")

datacontract/engines/soda/connections/kafka.py

@@ -6,6 +6,7 @@ import tempfile
 from datacontract.export.avro_converter import to_avro_schema_json
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Server
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum
 
 
 def create_spark_session():
@@ -16,7 +17,7 @@ def create_spark_session():
     except ImportError as e:
         raise DataContractException(
             type="schema",
-            result="failed",
+            result=ResultEnum.failed,
             name="pyspark is missing",
             reason="Install the extra datacontract-cli[kafka] to use kafka",
             engine="datacontract",
@@ -33,7 +34,7 @@ def create_spark_session():
         .config("spark.ui.enabled", "false")
         .config(
             "spark.jars.packages",
-            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.2,org.apache.spark:spark-avro_2.12:3.5.2",
+            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5,org.apache.spark:spark-avro_2.12:3.5.5",
         )
         .getOrCreate()
     )

datacontract/export/avro_converter.py

@@ -91,7 +91,9 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         if field.precision is not None:
             typeVal["precision"] = field.precision
         return typeVal
-    elif field.type in ["float", "double"]:
+    elif field.type in ["float"]:
+        return "float"
+    elif field.type in ["double"]:
         return "double"
     elif field.type in ["integer", "int"]:
         return "int"
@@ -107,6 +109,11 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         return {"type": "int", "logicalType": "date"}
     elif field.type in ["time"]:
         return "long"
+    elif field.type in ["map"]:
+        if field.config is not None and "values" in field.config:
+            return {"type": "map", "values": field.config["values"]}
+        else:
+            return "bytes"
     elif field.type in ["object", "record", "struct"]:
         if field.config is not None and "namespace" in field.config:
             return to_avro_record(field_name, field.fields, field.description, field.config["namespace"])

datacontract/export/bigquery_converter.py

@@ -103,7 +103,7 @@ def map_type_to_bigquery(field: Field) -> str:
     elif field_type.lower() == "date":
         return "DATE"
     elif field_type.lower() == "timestamp_ntz":
-        return "TIME"
+        return "DATETIME"
     elif field_type.lower() in ["number", "decimal", "numeric"]:
         return "NUMERIC"
     elif field_type.lower() == "double":

datacontract/export/duckdb_type_converter.py

@@ -0,0 +1,57 @@
+from typing import Dict
+
+from datacontract.model.data_contract_specification import Field
+
+
+# https://duckdb.org/docs/data/csv/overview.html
+# ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR']
+def convert_to_duckdb_csv_type(field) -> None | str:
+    datacontract_type = field.type
+    if datacontract_type is None:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["string", "varchar", "text"]:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if datacontract_type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP"
+    if datacontract_type.lower() in ["date"]:
+        return "DATE"
+    if datacontract_type.lower() in ["time"]:
+        return "TIME"
+    if datacontract_type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "VARCHAR"
+    if datacontract_type.lower() in ["float", "double"]:
+        return "DOUBLE"
+    if datacontract_type.lower() in ["integer", "int", "long", "bigint"]:
+        return "BIGINT"
+    if datacontract_type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if datacontract_type.lower() in ["object", "record", "struct"]:
+        # not supported in CSV
+        return "VARCHAR"
+    if datacontract_type.lower() in ["bytes"]:
+        # not supported in CSV
+        return "VARCHAR"
+    if datacontract_type.lower() in ["array"]:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["null"]:
+        return "SQLNULL"
+    return "VARCHAR"
+
+
+def convert_to_duckdb_json_type(field: Field) -> None | str:
+    datacontract_type = field.type
+    if datacontract_type is None:
+        return "VARCHAR"
+    if datacontract_type.lower() in ["array"]:
+        return convert_to_duckdb_json_type(field.items) + "[]"  # type: ignore
+    if datacontract_type.lower() in ["object", "record", "struct"]:
+        return convert_to_duckdb_object(field.fields)
+    return convert_to_duckdb_csv_type(field)
+
+
+def convert_to_duckdb_object(fields: Dict[str, Field]):
+    columns = [f'"{x[0]}" {convert_to_duckdb_json_type(x[1])}' for x in fields.items()]
+    return f"STRUCT({', '.join(columns)})"

datacontract/export/odcs_v3_exporter.py

@@ -13,12 +13,11 @@ class OdcsV3Exporter(Exporter):
 
 def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
     odcs = {
-        "apiVersion": "v3.0.0",
+        "apiVersion": "v3.0.1",
         "kind": "DataContract",
         "id": data_contract_spec.id,
         "name": data_contract_spec.info.title,
         "version": data_contract_spec.info.version,
-        "domain": data_contract_spec.info.owner,
         "status": to_status(data_contract_spec.info.status),
     }
 
@@ -126,13 +125,15 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
             odcs["servers"] = servers
 
     odcs["customProperties"] = []
+    if data_contract_spec.info.owner is not None:
+        odcs["customProperties"].append({"property": "owner", "value": data_contract_spec.info.owner})
     if data_contract_spec.info.model_extra is not None:
         for key, value in data_contract_spec.info.model_extra.items():
             odcs["customProperties"].append({"property": key, "value": value})
     if len(odcs["customProperties"]) == 0:
         del odcs["customProperties"]
 
-    return yaml.dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
+    return yaml.safe_dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
 
 
 def to_odcs_schema(model_key, model_value: Model) -> dict:
@@ -217,13 +218,13 @@ def to_property(field_name: str, field: Field) -> dict:
     if field.description is not None:
         property["description"] = field.description
     if field.required is not None:
-        property["nullable"] = not field.required
+        property["required"] = field.required
     if field.unique is not None:
         property["unique"] = field.unique
     if field.classification is not None:
         property["classification"] = field.classification
     if field.examples is not None:
-        property["examples"] = field.examples
+        property["examples"] = field.examples.copy()
     if field.example is not None:
         property["examples"] = [field.example]
     if field.primaryKey is not None and field.primaryKey: