datacontract-cli 0.10.14__py3-none-any.whl → 0.10.16__py3-none-any.whl

datacontract/engines/datacontract/check_that_datacontract_file_exists.py

@@ -1,6 +1,6 @@
 import os
 
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, Run
 
 
 def check_that_datacontract_file_exists(run: Run, file_path: str):

datacontract/engines/fastjsonschema/check_jsonschema.py

@@ -1,31 +1,114 @@
 import json
 import logging
 import os
+import threading
+from typing import List, Optional
 
 import fastjsonschema
+from fastjsonschema import JsonSchemaValueException
 
 from datacontract.engines.fastjsonschema.s3.s3_read_files import yield_s3_files
 from datacontract.export.jsonschema_converter import to_jsonschema
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, Run
 
+# Thread-safe cache for primaryKey fields.
+_primary_key_cache = {}
+_cache_lock = threading.Lock()
 
-def validate_json_stream(model_name, validate, json_stream):
+
+def get_primary_key_field(schema: dict, model_name: str) -> Optional[str]:
+    # Check cache first.
+    with _cache_lock:
+        cached_value = _primary_key_cache.get(model_name)
+        if cached_value is not None:
+            return cached_value
+
+    # Find primaryKey field.
+    fields = schema.get("properties", {})
+    for field_name, attributes in fields.items():
+        if attributes.get("primaryKey", False):
+            # Cache the result before returning.
+            with _cache_lock:
+                _primary_key_cache[model_name] = field_name
+            return field_name
+
+    # Return None if no primary key was found.
+    return None
+
+
+def get_primary_key_value(schema: dict, model_name: str, json_object: dict) -> Optional[str]:
+    # Get the `primaryKey` field.
+    primary_key_field = get_primary_key_field(schema, model_name)
+    if not primary_key_field:
+        return None
+
+    # Return the value of the `primaryKey` field in the JSON object.
+    return json_object.get(primary_key_field)
+
+
+def process_exceptions(run, exceptions: List[DataContractException]):
+    if not exceptions:
+        return
+
+    # Define the maximum number of errors to process (can be adjusted by defining an ENV variable).
     try:
-        logging.info("Validating JSON")
-        for json_obj in json_stream:
+        error_limit = int(os.getenv("DATACONTRACT_MAX_ERRORS", 500))
+    except ValueError:
+        # Fallback to default if environment variable is invalid.
+        error_limit = 500
+
+    # Calculate the effective limit to avoid index out of range
+    limit = min(len(exceptions), error_limit)
+
+    # Add all exceptions up to the limit - 1 to `run.checks`.
+    DEFAULT_ERROR_MESSAGE = "An error occurred during validation phase. See the logs for more details."
+    run.checks.extend(
+        [
+            Check(
+                type=exception.type,
+                name=exception.name,
+                result=exception.result,
+                reason=exception.reason,
+                model=exception.model,
+                engine=exception.engine,
+                message=exception.message or DEFAULT_ERROR_MESSAGE,
+            )
+            for exception in exceptions[: limit - 1]
+        ]
+    )
+
+    # Raise the last exception within the limit.
+    last_exception = exceptions[limit - 1]
+    raise last_exception
+
+
+def validate_json_stream(
+    schema: dict, model_name: str, validate: callable, json_stream: list[dict]
+) -> List[DataContractException]:
+    logging.info(f"Validating JSON stream for model: '{model_name}'.")
+    exceptions: List[DataContractException] = []
+    for json_obj in json_stream:
+        try:
             validate(json_obj)
-        return True
-    except fastjsonschema.JsonSchemaValueException as e:
-        raise DataContractException(
-            type="schema",
-            name="Check that JSON has valid schema",
-            model=model_name,
-            reason=e.message,
-            engine="jsonschema",
-            original_exception=e,
-        )
+        except JsonSchemaValueException as e:
+            logging.warning(f"Validation failed for JSON object with type: '{model_name}'.")
+            primary_key_value = get_primary_key_value(schema, model_name, json_obj)
+            exceptions.append(
+                DataContractException(
+                    type="schema",
+                    name="Check that JSON has valid schema",
+                    result="failed",
+                    reason=f"{f'#{primary_key_value}: ' if primary_key_value is not None else ''}{e.message}",
+                    model=model_name,
+                    engine="jsonschema",
+                    message=e.message,
+                )
+            )
+    if not exceptions:
+        logging.info(f"All JSON objects in the stream passed validation for model: '{model_name}'.")
+    return exceptions
 
 
 def read_json_lines(file):
@@ -59,17 +142,22 @@ def read_json_file_content(file_content: str):
     yield json.loads(file_content)
 
 
-def process_json_file(run, model_name, validate, file, delimiter):
+def process_json_file(run, schema, model_name, validate, file, delimiter):
     if delimiter == "new_line":
         json_stream = read_json_lines(file)
     elif delimiter == "array":
         json_stream = read_json_array(file)
     else:
         json_stream = read_json_file(file)
-    validate_json_stream(model_name, validate, json_stream)
 
+    # Validate the JSON stream and collect exceptions.
+    exceptions = validate_json_stream(schema, model_name, validate, json_stream)
 
-def process_local_file(run, server, model_name, validate):
+    # Handle all errors from schema validation.
+    process_exceptions(run, exceptions)
+
+
+def process_local_file(run, server, schema, model_name, validate):
     path = server.path
     if "{model}" in path:
         path = path.format(model=model_name)
@@ -79,7 +167,7 @@ def process_local_file(run, server, model_name, validate):
     else:
         logging.info(f"Processing file {path}")
         with open(path, "r") as file:
-            process_json_file(run, model_name, validate, file, server.delimiter)
+            process_json_file(run, schema, model_name, validate, file, server.delimiter)
 
 
 def process_directory(run, path, server, model_name, validate):
@@ -94,7 +182,7 @@ def process_directory(run, path, server, model_name, validate):
     return success
 
 
-def process_s3_file(server, model_name, validate):
+def process_s3_file(run, server, schema, model_name, validate):
     s3_endpoint_url = server.endpointUrl
     s3_location = server.location
     if "{model}" in s3_location:
@@ -118,7 +206,11 @@ def process_s3_file(server, model_name, validate):
             engine="datacontract",
         )
 
-    return validate_json_stream(model_name, validate, json_stream)
+    # Validate the JSON stream and collect exceptions.
+    exceptions = validate_json_stream(schema, model_name, validate, json_stream)
+
+    # Handle all errors from schema validation.
+    process_exceptions(run, exceptions)
 
 
 def check_jsonschema(run: Run, data_contract: DataContractSpecification, server: Server):
@@ -155,9 +247,9 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
 
         # Process files based on server type
         if server.type == "local":
-            process_local_file(run, server, model_name, validate)
+            process_local_file(run, server, schema, model_name, validate)
         elif server.type == "s3":
-            process_s3_file(server, model_name, validate)
+            process_s3_file(run, server, schema, model_name, validate)
         elif server.type == "gcs":
             run.checks.append(
                 Check(

datacontract/engines/soda/check_soda_execute.py

@@ -12,7 +12,7 @@ from datacontract.engines.soda.connections.sqlserver import to_sqlserver_soda_co
 from datacontract.engines.soda.connections.trino import to_trino_soda_configuration
 from datacontract.export.sodacl_converter import to_sodacl_yaml
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
-from datacontract.model.run import Run, Check, Log
+from datacontract.model.run import Check, Log, ResultEnum, Run
 
 
 def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
@@ -33,7 +33,7 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
                 Check(
                     type="general",
                     name="Check that format is supported",
-                    result="warning",
+                    result=ResultEnum.warning,
                     reason=f"Format {server.format} not yet supported by datacontract CLI",
                     engine="datacontract",
                 )
@@ -93,7 +93,7 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
             Check(
                 type="general",
                 name="Check that server type is supported",
-                result="warning",
+                result=ResultEnum.warning,
                 reason=f"Server type {server.type} not yet supported by datacontract CLI",
                 engine="datacontract-cli",
             )
@@ -176,9 +176,11 @@ def update_reason(check, c):
         if block["title"] == "Diagnostics":
             # Extract and print the 'text' value
             diagnostics_text = block["text"]
-            print(diagnostics_text)
+            # print(diagnostics_text)
             diagnostics_text_split = diagnostics_text.split(":icon-fail: ")
             if len(diagnostics_text_split) > 1:
                 check.reason = diagnostics_text_split[1].strip()
-                print(check.reason)
+                # print(check.reason)
             break  # Exit the loop once the desired block is found
+    if "fail" in c["diagnostics"]:
+        check.reason = f"Got: {c['diagnostics']['value']} Expected: {c['diagnostics']['fail']}"

datacontract/engines/soda/connections/duckdb.py

@@ -1,6 +1,7 @@
 import os
 
 import duckdb
+
 from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
 from datacontract.model.run import Run
 

datacontract/engines/soda/connections/kafka.py

@@ -2,7 +2,7 @@ import logging
 import os
 
 from datacontract.export.avro_converter import to_avro_schema_json
-from datacontract.model.data_contract_specification import DataContractSpecification, Server, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Server
 from datacontract.model.exceptions import DataContractException
 
 
@@ -69,8 +69,8 @@ def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Se
 
 def process_avro_format(df, model_name, model):
     try:
-        from pyspark.sql.functions import col, expr
         from pyspark.sql.avro.functions import from_avro
+        from pyspark.sql.functions import col, expr
     except ImportError as e:
         raise DataContractException(
             type="schema",
@@ -167,21 +167,21 @@ def to_struct_type(fields):
 def to_struct_field(field_name: str, field: Field):
     try:
         from pyspark.sql.types import (
-            StructType,
-            StructField,
-            StringType,
+            ArrayType,
+            BinaryType,
+            BooleanType,
+            DataType,
+            DateType,
             DecimalType,
             DoubleType,
             IntegerType,
             LongType,
-            BooleanType,
-            TimestampType,
-            TimestampNTZType,
-            DateType,
-            BinaryType,
-            ArrayType,
             NullType,
-            DataType,
+            StringType,
+            StructField,
+            StructType,
+            TimestampNTZType,
+            TimestampType,
         )
     except ImportError as e:
         raise DataContractException(

datacontract/export/avro_idl_converter.py

@@ -3,12 +3,11 @@ from dataclasses import dataclass
 from enum import Enum
 from io import StringIO
 
+from datacontract.export.exporter import Exporter
 from datacontract.lint.resolve import inline_definitions_into_data_contract
 from datacontract.model.data_contract_specification import DataContractSpecification, Field
 from datacontract.model.exceptions import DataContractException
 
-from datacontract.export.exporter import Exporter
-
 
 class AvroPrimitiveType(Enum):
     int = "int"

datacontract/export/bigquery_converter.py

@@ -2,10 +2,9 @@ import json
 import logging
 from typing import Dict, List
 
-from datacontract.model.data_contract_specification import Model, Field, Server
-from datacontract.model.exceptions import DataContractException
-
 from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.model.data_contract_specification import Field, Model, Server
+from datacontract.model.exceptions import DataContractException
 
 
 class BigQueryExporter(Exporter):
@@ -109,6 +108,8 @@ def map_type_to_bigquery(field: Field) -> str:
         return "NUMERIC"
     elif field_type.lower() == "double":
         return "BIGNUMERIC"
+    elif field_type.lower() in ["object", "record"] and not field.fields:
+        return "JSON"
     elif field_type.lower() in ["object", "record", "array"]:
         return "RECORD"
     elif field_type.lower() == "struct":

datacontract/export/data_caterer_converter.py

@@ -3,7 +3,7 @@ from typing import Dict
 import yaml
 
 from datacontract.export.exporter import Exporter
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Server
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server
 
 
 class DataCatererExporter(Exporter):

datacontract/export/dbml_converter.py

@@ -3,13 +3,11 @@ from importlib.metadata import version
 from typing import Tuple
 
 import pytz
-from datacontract.model.exceptions import DataContractException
 
 import datacontract.model.data_contract_specification as spec
-from datacontract.export.sql_type_converter import convert_to_sql_type
-
-
 from datacontract.export.exporter import Exporter
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.exceptions import DataContractException
 
 
 class DbmlExporter(Exporter):

datacontract/export/dbt_converter.py

@@ -1,11 +1,10 @@
-from typing import Dict
+from typing import Dict, Optional
 
 import yaml
 
-from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
-
 from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
 
 class DbtExporter(Exporter):
@@ -53,14 +52,14 @@ def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name
         # TODO escape SQL reserved key words, probably dependent on server type
         columns.append(field_name)
     return f"""
-    select 
+    select
         {", ".join(columns)}
     from {{{{ source('{id}', '{model_name}') }}}}
 """
 
 
 def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: str = None):
-    source = {"name": data_contract_spec.id, "tables": []}
+    source = {"name": data_contract_spec.id}
     dbt = {
         "version": 2,
         "sources": [source],
@@ -70,24 +69,31 @@ def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: s
     if data_contract_spec.info.description is not None:
         source["description"] = data_contract_spec.info.description
     found_server = data_contract_spec.servers.get(server)
+    adapter_type = None
     if found_server is not None:
-        source["database"] = found_server.database
-        source["schema"] = found_server.schema_
+        adapter_type = found_server.type
+        if adapter_type == "bigquery":
+            source["database"] = found_server.project
+            source["schema"] = found_server.dataset
+        else:
+            source["database"] = found_server.database
+            source["schema"] = found_server.schema_
 
+    source["tables"] = []
     for model_key, model_value in data_contract_spec.models.items():
-        dbt_model = _to_dbt_source_table(model_key, model_value)
+        dbt_model = _to_dbt_source_table(model_key, model_value, adapter_type)
         source["tables"].append(dbt_model)
     return yaml.dump(dbt, indent=2, sort_keys=False, allow_unicode=True)
 
 
-def _to_dbt_source_table(model_key, model_value: Model) -> dict:
+def _to_dbt_source_table(model_key, model_value: Model, adapter_type: Optional[str]) -> dict:
     dbt_model = {
         "name": model_key,
     }
 
     if model_value.description is not None:
         dbt_model["description"] = model_value.description
-    columns = _to_columns(model_value.fields, False, False)
+    columns = _to_columns(model_value.fields, False, adapter_type)
     if columns:
         dbt_model["columns"] = columns
     return dbt_model
@@ -108,7 +114,7 @@ def _to_dbt_model(model_key, model_value: Model, data_contract_spec: DataContrac
         dbt_model["config"]["contract"] = {"enforced": True}
     if model_value.description is not None:
         dbt_model["description"] = model_value.description
-    columns = _to_columns(model_value.fields, _supports_constraints(model_type), True)
+    columns = _to_columns(model_value.fields, _supports_constraints(model_type), None)
     if columns:
         dbt_model["columns"] = columns
     return dbt_model
@@ -131,48 +137,47 @@ def _supports_constraints(model_type):
     return model_type == "table" or model_type == "incremental"
 
 
-def _to_columns(fields: Dict[str, Field], supports_constraints: bool, supports_datatype: bool) -> list:
+def _to_columns(fields: Dict[str, Field], supports_constraints: bool, adapter_type: Optional[str]) -> list:
     columns = []
     for field_name, field in fields.items():
-        column = _to_column(field, supports_constraints, supports_datatype)
-        column["name"] = field_name
+        column = _to_column(field_name, field, supports_constraints, adapter_type)
         columns.append(column)
     return columns
 
 
-def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool) -> dict:
-    column = {}
-    dbt_type = convert_to_sql_type(field, "snowflake")
+def _to_column(field_name: str, field: Field, supports_constraints: bool, adapter_type: Optional[str]) -> dict:
+    column = {"name": field_name}
+    adapter_type = adapter_type or "snowflake"
+    dbt_type = convert_to_sql_type(field, adapter_type)
+
+    column["data_tests"] = []
     if dbt_type is not None:
-        if supports_datatype:
-            column["data_type"] = dbt_type
-        else:
-            column.setdefault("tests", []).append(
-                {"dbt_expectations.dbt_expectations.expect_column_values_to_be_of_type": {"column_type": dbt_type}}
-            )
+        column["data_type"] = dbt_type
+    else:
+        column["data_tests"].append(
+            {"dbt_expectations.dbt_expectations.expect_column_values_to_be_of_type": {"column_type": dbt_type}}
+        )
     if field.description is not None:
         column["description"] = field.description
     if field.required:
         if supports_constraints:
             column.setdefault("constraints", []).append({"type": "not_null"})
         else:
-            column.setdefault("tests", []).append("not_null")
+            column["data_tests"].append("not_null")
     if field.unique:
         if supports_constraints:
             column.setdefault("constraints", []).append({"type": "unique"})
         else:
-            column.setdefault("tests", []).append("unique")
+            column["data_tests"].append("unique")
     if field.enum is not None and len(field.enum) > 0:
-        column.setdefault("tests", []).append({"accepted_values": {"values": field.enum}})
+        column["data_tests"].append({"accepted_values": {"values": field.enum}})
     if field.minLength is not None or field.maxLength is not None:
         length_test = {}
         if field.minLength is not None:
             length_test["min_value"] = field.minLength
         if field.maxLength is not None:
             length_test["max_value"] = field.maxLength
-        column.setdefault("tests", []).append(
-            {"dbt_expectations.expect_column_value_lengths_to_be_between": length_test}
-        )
+        column["data_tests"].append({"dbt_expectations.expect_column_value_lengths_to_be_between": length_test})
     if field.pii is not None:
         column.setdefault("meta", {})["pii"] = field.pii
     if field.classification is not None:
@@ -181,9 +186,7 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
         column.setdefault("tags", []).extend(field.tags)
     if field.pattern is not None:
         # Beware, the data contract pattern is a regex, not a like pattern
-        column.setdefault("tests", []).append(
-            {"dbt_expectations.expect_column_values_to_match_regex": {"regex": field.pattern}}
-        )
+        column["data_tests"].append({"dbt_expectations.expect_column_values_to_match_regex": {"regex": field.pattern}})
     if (
         field.minimum is not None
         or field.maximum is not None
@@ -195,7 +198,7 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
             range_test["min_value"] = field.minimum
         if field.maximum is not None:
             range_test["max_value"] = field.maximum
-        column.setdefault("tests", []).append({"dbt_expectations.expect_column_values_to_be_between": range_test})
+        column["data_tests"].append({"dbt_expectations.expect_column_values_to_be_between": range_test})
     elif (
         field.exclusiveMinimum is not None
         or field.exclusiveMaximum is not None
@@ -208,18 +211,18 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
         if field.exclusiveMaximum is not None:
             range_test["max_value"] = field.exclusiveMaximum
         range_test["strictly"] = True
-        column.setdefault("tests", []).append({"dbt_expectations.expect_column_values_to_be_between": range_test})
+        column["data_tests"].append({"dbt_expectations.expect_column_values_to_be_between": range_test})
     else:
         if field.minimum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {"dbt_expectations.expect_column_values_to_be_between": {"min_value": field.minimum}}
             )
         if field.maximum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {"dbt_expectations.expect_column_values_to_be_between": {"max_value": field.maximum}}
             )
         if field.exclusiveMinimum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {
                     "dbt_expectations.expect_column_values_to_be_between": {
                         "min_value": field.exclusiveMinimum,
@@ -228,7 +231,7 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
                 }
             )
         if field.exclusiveMaximum is not None:
-            column.setdefault("tests", []).append(
+            column["data_tests"].append(
                 {
                     "dbt_expectations.expect_column_values_to_be_between": {
                         "max_value": field.exclusiveMaximum,
@@ -237,5 +240,8 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
                 }
             )
 
+        if not column["data_tests"]:
+            column.pop("data_tests")
+
     # TODO: all constraints
     return column

datacontract/export/exporter.py

@@ -1,6 +1,6 @@
+import typing
 from abc import ABC, abstractmethod
 from enum import Enum
-import typing
 
 from datacontract.model.data_contract_specification import DataContractSpecification
 
@@ -40,6 +40,7 @@ class ExportFormat(str, Enum):
     sqlalchemy = "sqlalchemy"
     data_caterer = "data-caterer"
     dcs = "dcs"
+    iceberg = "iceberg"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/export/exporter_factory.py

@@ -1,6 +1,7 @@
 import importlib
 import sys
-from datacontract.export.exporter import ExportFormat, Exporter
+
+from datacontract.export.exporter import Exporter, ExportFormat
 
 
 class ExporterFactory:
@@ -117,7 +118,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.great_expectations,
     module_path="datacontract.export.great_expectations_converter",
-    class_name="GreateExpectationsExporter",
+    class_name="GreatExpectationsExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -167,3 +168,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.dcs, module_path="datacontract.export.dcs_exporter", class_name="DcsExporter"
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
+)

datacontract/export/go_converter.py

@@ -1,6 +1,7 @@
-import datacontract.model.data_contract_specification as spec
-from typing import List
 import re
+from typing import List
+
+import datacontract.model.data_contract_specification as spec
 from datacontract.export.exporter import Exporter