datacontract-cli 0.10.13__py3-none-any.whl → 0.10.15__py3-none-any.whl

datacontract/data_contract.py

@@ -8,7 +8,12 @@ import yaml
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
-from datacontract.breaking.breaking import models_breaking_changes, quality_breaking_changes
+from datacontract.breaking.breaking import (
+    info_breaking_changes,
+    models_breaking_changes,
+    quality_breaking_changes,
+    terms_breaking_changes,
+)
 from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
     check_that_datacontract_contains_valid_server_configuration,
 )
@@ -17,7 +22,6 @@ from datacontract.engines.soda.check_soda_execute import check_soda_execute
 from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
-
 from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
 from datacontract.integration.opentelemetry import publish_test_results_to_opentelemetry
 from datacontract.lint import resolve
@@ -28,10 +32,10 @@ from datacontract.lint.linters.field_reference_linter import FieldReferenceLinte
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
 from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
-from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Severity
+from datacontract.model.breaking_change import BreakingChange, BreakingChanges, Severity
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, Run
 
 
 class DataContract:
@@ -199,7 +203,15 @@ class DataContract:
 
         except DataContractException as e:
             run.checks.append(
-                Check(type=e.type, result=e.result, name=e.name, reason=e.reason, engine=e.engine, details="")
+                Check(
+                    type=e.type,
+                    name=e.name,
+                    result=e.result,
+                    reason=e.reason,
+                    model=e.model,
+                    engine=e.engine,
+                    details="",
+                )
             )
             run.log_error(str(e))
         except Exception as e:
@@ -268,6 +280,24 @@ class DataContract:
 
         breaking_changes = list[BreakingChange]()
 
+        breaking_changes.extend(
+            info_breaking_changes(
+                old_info=old.info,
+                new_info=new.info,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
+
+        breaking_changes.extend(
+            terms_breaking_changes(
+                old_terms=old.terms,
+                new_terms=new.terms,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
+
         breaking_changes.extend(
             quality_breaking_changes(
                 old_quality=old.quality,

datacontract/engines/datacontract/check_that_datacontract_file_exists.py

@@ -1,6 +1,6 @@
 import os
 
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, Run
 
 
 def check_that_datacontract_file_exists(run: Run, file_path: str):

datacontract/engines/fastjsonschema/check_jsonschema.py

@@ -1,31 +1,114 @@
 import json
 import logging
 import os
+import threading
+from typing import List, Optional
 
 import fastjsonschema
+from fastjsonschema import JsonSchemaValueException
 
 from datacontract.engines.fastjsonschema.s3.s3_read_files import yield_s3_files
 from datacontract.export.jsonschema_converter import to_jsonschema
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, Run
 
+# Thread-safe cache for primaryKey fields.
+_primary_key_cache = {}
+_cache_lock = threading.Lock()
 
-def validate_json_stream(model_name, validate, json_stream):
+
+def get_primary_key_field(schema: dict, model_name: str) -> Optional[str]:
+    # Check cache first.
+    with _cache_lock:
+        cached_value = _primary_key_cache.get(model_name)
+        if cached_value is not None:
+            return cached_value
+
+    # Find primaryKey field.
+    fields = schema.get("properties", {})
+    for field_name, attributes in fields.items():
+        if attributes.get("primaryKey", False):
+            # Cache the result before returning.
+            with _cache_lock:
+                _primary_key_cache[model_name] = field_name
+            return field_name
+
+    # Return None if no primary key was found.
+    return None
+
+
+def get_primary_key_value(schema: dict, model_name: str, json_object: dict) -> Optional[str]:
+    # Get the `primaryKey` field.
+    primary_key_field = get_primary_key_field(schema, model_name)
+    if not primary_key_field:
+        return None
+
+    # Return the value of the `primaryKey` field in the JSON object.
+    return json_object.get(primary_key_field)
+
+
+def process_exceptions(run, exceptions: List[DataContractException]):
+    if not exceptions:
+        return
+
+    # Define the maximum number of errors to process (can be adjusted by defining an ENV variable).
     try:
-        logging.info("Validating JSON")
-        for json_obj in json_stream:
+        error_limit = int(os.getenv("DATACONTRACT_MAX_ERRORS", 500))
+    except ValueError:
+        # Fallback to default if environment variable is invalid.
+        error_limit = 500
+
+    # Calculate the effective limit to avoid index out of range
+    limit = min(len(exceptions), error_limit)
+
+    # Add all exceptions up to the limit - 1 to `run.checks`.
+    DEFAULT_ERROR_MESSAGE = "An error occurred during validation phase. See the logs for more details."
+    run.checks.extend(
+        [
+            Check(
+                type=exception.type,
+                name=exception.name,
+                result=exception.result,
+                reason=exception.reason,
+                model=exception.model,
+                engine=exception.engine,
+                message=exception.message or DEFAULT_ERROR_MESSAGE,
+            )
+            for exception in exceptions[: limit - 1]
+        ]
+    )
+
+    # Raise the last exception within the limit.
+    last_exception = exceptions[limit - 1]
+    raise last_exception
+
+
+def validate_json_stream(
+    schema: dict, model_name: str, validate: callable, json_stream: list[dict]
+) -> List[DataContractException]:
+    logging.info(f"Validating JSON stream for model: '{model_name}'.")
+    exceptions: List[DataContractException] = []
+    for json_obj in json_stream:
+        try:
             validate(json_obj)
-        return True
-    except fastjsonschema.JsonSchemaValueException as e:
-        raise DataContractException(
-            type="schema",
-            name="Check that JSON has valid schema",
-            model=model_name,
-            reason=e.message,
-            engine="jsonschema",
-            original_exception=e,
-        )
+        except JsonSchemaValueException as e:
+            logging.warning(f"Validation failed for JSON object with type: '{model_name}'.")
+            primary_key_value = get_primary_key_value(schema, model_name, json_obj)
+            exceptions.append(
+                DataContractException(
+                    type="schema",
+                    name="Check that JSON has valid schema",
+                    result="failed",
+                    reason=f"{f'#{primary_key_value}: ' if primary_key_value is not None else ''}{e.message}",
+                    model=model_name,
+                    engine="jsonschema",
+                    message=e.message,
+                )
+            )
+    if not exceptions:
+        logging.info(f"All JSON objects in the stream passed validation for model: '{model_name}'.")
+    return exceptions
 
 
 def read_json_lines(file):
@@ -59,17 +142,22 @@ def read_json_file_content(file_content: str):
     yield json.loads(file_content)
 
 
-def process_json_file(run, model_name, validate, file, delimiter):
+def process_json_file(run, schema, model_name, validate, file, delimiter):
     if delimiter == "new_line":
         json_stream = read_json_lines(file)
     elif delimiter == "array":
         json_stream = read_json_array(file)
     else:
         json_stream = read_json_file(file)
-    validate_json_stream(model_name, validate, json_stream)
 
+    # Validate the JSON stream and collect exceptions.
+    exceptions = validate_json_stream(schema, model_name, validate, json_stream)
 
-def process_local_file(run, server, model_name, validate):
+    # Handle all errors from schema validation.
+    process_exceptions(run, exceptions)
+
+
+def process_local_file(run, server, schema, model_name, validate):
     path = server.path
     if "{model}" in path:
         path = path.format(model=model_name)
@@ -79,7 +167,7 @@ def process_local_file(run, server, model_name, validate):
     else:
         logging.info(f"Processing file {path}")
         with open(path, "r") as file:
-            process_json_file(run, model_name, validate, file, server.delimiter)
+            process_json_file(run, schema, model_name, validate, file, server.delimiter)
 
 
 def process_directory(run, path, server, model_name, validate):
@@ -94,7 +182,7 @@ def process_directory(run, path, server, model_name, validate):
     return success
 
 
-def process_s3_file(server, model_name, validate):
+def process_s3_file(run, server, schema, model_name, validate):
     s3_endpoint_url = server.endpointUrl
     s3_location = server.location
     if "{model}" in s3_location:
@@ -118,7 +206,11 @@ def process_s3_file(server, model_name, validate):
             engine="datacontract",
         )
 
-    return validate_json_stream(model_name, validate, json_stream)
+    # Validate the JSON stream and collect exceptions.
+    exceptions = validate_json_stream(schema, model_name, validate, json_stream)
+
+    # Handle all errors from schema validation.
+    process_exceptions(run, exceptions)
 
 
 def check_jsonschema(run: Run, data_contract: DataContractSpecification, server: Server):
@@ -155,9 +247,9 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
 
         # Process files based on server type
         if server.type == "local":
-            process_local_file(run, server, model_name, validate)
+            process_local_file(run, server, schema, model_name, validate)
         elif server.type == "s3":
-            process_s3_file(server, model_name, validate)
+            process_s3_file(run, server, schema, model_name, validate)
         elif server.type == "gcs":
             run.checks.append(
                 Check(

datacontract/engines/soda/check_soda_execute.py

@@ -12,7 +12,7 @@ from datacontract.engines.soda.connections.sqlserver import to_sqlserver_soda_co
 from datacontract.engines.soda.connections.trino import to_trino_soda_configuration
 from datacontract.export.sodacl_converter import to_sodacl_yaml
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
-from datacontract.model.run import Run, Check, Log
+from datacontract.model.run import Check, Log, Run
 
 
 def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
@@ -176,9 +176,11 @@ def update_reason(check, c):
         if block["title"] == "Diagnostics":
             # Extract and print the 'text' value
             diagnostics_text = block["text"]
-            print(diagnostics_text)
+            # print(diagnostics_text)
             diagnostics_text_split = diagnostics_text.split(":icon-fail: ")
             if len(diagnostics_text_split) > 1:
                 check.reason = diagnostics_text_split[1].strip()
-                print(check.reason)
+                # print(check.reason)
             break  # Exit the loop once the desired block is found
+    if c["diagnostics"]["fail"] is not None:
+        check.reason = f"Got: {c['diagnostics']['value']} Expected: {c['diagnostics']['fail']}"

datacontract/engines/soda/connections/duckdb.py

@@ -1,6 +1,7 @@
 import os
 
 import duckdb
+
 from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
 from datacontract.model.run import Run
 

datacontract/engines/soda/connections/kafka.py

@@ -2,7 +2,7 @@ import logging
 import os
 
 from datacontract.export.avro_converter import to_avro_schema_json
-from datacontract.model.data_contract_specification import DataContractSpecification, Server, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Server
 from datacontract.model.exceptions import DataContractException
 
 
@@ -69,8 +69,8 @@ def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Se
 
 def process_avro_format(df, model_name, model):
     try:
-        from pyspark.sql.functions import col, expr
         from pyspark.sql.avro.functions import from_avro
+        from pyspark.sql.functions import col, expr
     except ImportError as e:
         raise DataContractException(
             type="schema",
@@ -112,17 +112,38 @@ def get_auth_options():
     """Retrieve Kafka authentication options from environment variables."""
     kafka_sasl_username = os.getenv("DATACONTRACT_KAFKA_SASL_USERNAME")
     kafka_sasl_password = os.getenv("DATACONTRACT_KAFKA_SASL_PASSWORD")
+    kafka_sasl_mechanism = os.getenv("DATACONTRACT_KAFKA_SASL_MECHANISM", "PLAIN").upper()
 
-    if kafka_sasl_username is None or kafka_sasl_username == "":
+    # Skip authentication if credentials are not provided
+    if not kafka_sasl_username or not kafka_sasl_password:
         return {}
 
-    return {
-        "kafka.sasl.mechanism": "PLAIN",
-        "kafka.security.protocol": "SASL_SSL",
-        "kafka.sasl.jaas.config": (
+    # SASL mechanisms supported by Kafka
+    jaas_config = {
+        "PLAIN": (
             f"org.apache.kafka.common.security.plain.PlainLoginModule required "
             f'username="{kafka_sasl_username}" password="{kafka_sasl_password}";'
         ),
+        "SCRAM-SHA-256": (
+            f"org.apache.kafka.common.security.scram.ScramLoginModule required "
+            f'username="{kafka_sasl_username}" password="{kafka_sasl_password}";'
+        ),
+        "SCRAM-SHA-512": (
+            f"org.apache.kafka.common.security.scram.ScramLoginModule required "
+            f'username="{kafka_sasl_username}" password="{kafka_sasl_password}";'
+        ),
+        # Add more mechanisms as needed
+    }
+
+    # Validate SASL mechanism
+    if kafka_sasl_mechanism not in jaas_config:
+        raise ValueError(f"Unsupported SASL mechanism: {kafka_sasl_mechanism}")
+
+    # Return config
+    return {
+        "kafka.sasl.mechanism": kafka_sasl_mechanism,
+        "kafka.security.protocol": "SASL_SSL",
+        "kafka.sasl.jaas.config": jaas_config[kafka_sasl_mechanism],
     }
 
 
@@ -146,21 +167,21 @@ def to_struct_type(fields):
 def to_struct_field(field_name: str, field: Field):
     try:
         from pyspark.sql.types import (
-            StructType,
-            StructField,
-            StringType,
+            ArrayType,
+            BinaryType,
+            BooleanType,
+            DataType,
+            DateType,
             DecimalType,
             DoubleType,
             IntegerType,
             LongType,
-            BooleanType,
-            TimestampType,
-            TimestampNTZType,
-            DateType,
-            BinaryType,
-            ArrayType,
             NullType,
-            DataType,
+            StringType,
+            StructField,
+            StructType,
+            TimestampNTZType,
+            TimestampType,
         )
     except ImportError as e:
         raise DataContractException(

datacontract/export/avro_converter.py

@@ -81,9 +81,16 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         return "null"
     if field.type in ["string", "varchar", "text"]:
         return "string"
-    elif field.type in ["number", "decimal", "numeric"]:
+    elif field.type in ["number", "numeric"]:
         # https://avro.apache.org/docs/1.11.1/specification/#decimal
         return "bytes"
+    elif field.type in ["decimal"]:
+        typeVal = {"type": "bytes", "logicalType": "decimal"}
+        if field.scale is not None:
+            typeVal["scale"] = field.scale
+        if field.precision is not None:
+            typeVal["precision"] = field.precision
+        return typeVal
     elif field.type in ["float", "double"]:
         return "double"
     elif field.type in ["integer", "int"]:

datacontract/export/avro_idl_converter.py

@@ -3,12 +3,11 @@ from dataclasses import dataclass
 from enum import Enum
 from io import StringIO
 
+from datacontract.export.exporter import Exporter
 from datacontract.lint.resolve import inline_definitions_into_data_contract
 from datacontract.model.data_contract_specification import DataContractSpecification, Field
 from datacontract.model.exceptions import DataContractException
 
-from datacontract.export.exporter import Exporter
-
 
 class AvroPrimitiveType(Enum):
     int = "int"
@@ -64,6 +63,7 @@ class AvroIDLProtocol:
     model_types: list[AvroModelType]
 
 
+# TODO use DATACONTRACT_TYPES from datacontract/model/data_contract_specification.py
 avro_primitive_types = set(
     [
         "string",

datacontract/export/bigquery_converter.py

@@ -2,10 +2,9 @@ import json
 import logging
 from typing import Dict, List
 
-from datacontract.model.data_contract_specification import Model, Field, Server
-from datacontract.model.exceptions import DataContractException
-
 from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.model.data_contract_specification import Field, Model, Server
+from datacontract.model.exceptions import DataContractException
 
 
 class BigQueryExporter(Exporter):
@@ -109,6 +108,8 @@ def map_type_to_bigquery(field: Field) -> str:
         return "NUMERIC"
     elif field_type.lower() == "double":
         return "BIGNUMERIC"
+    elif field_type.lower() in ["object", "record"] and not field.fields:
+        return "JSON"
     elif field_type.lower() in ["object", "record", "array"]:
         return "RECORD"
     elif field_type.lower() == "struct":

datacontract/export/data_caterer_converter.py

@@ -3,7 +3,7 @@ from typing import Dict
 import yaml
 
 from datacontract.export.exporter import Exporter
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Server
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server
 
 
 class DataCatererExporter(Exporter):

datacontract/export/dbml_converter.py

@@ -3,13 +3,11 @@ from importlib.metadata import version
 from typing import Tuple
 
 import pytz
-from datacontract.model.exceptions import DataContractException
 
 import datacontract.model.data_contract_specification as spec
-from datacontract.export.sql_type_converter import convert_to_sql_type
-
-
 from datacontract.export.exporter import Exporter
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.exceptions import DataContractException
 
 
 class DbmlExporter(Exporter):

datacontract/export/dbt_converter.py

@@ -2,10 +2,9 @@ from typing import Dict
 
 import yaml
 
-from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
-
 from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
 
 class DbtExporter(Exporter):

datacontract/export/dcs_exporter.py

@@ -0,0 +1,6 @@
+from datacontract.export.exporter import Exporter
+
+
+class DcsExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return data_contract.to_yaml()

datacontract/export/exporter.py

@@ -1,6 +1,6 @@
+import typing
 from abc import ABC, abstractmethod
 from enum import Enum
-import typing
 
 from datacontract.model.data_contract_specification import DataContractSpecification
 
@@ -10,7 +10,7 @@ class Exporter(ABC):
         self.export_format = export_format
 
     @abstractmethod
-    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict | str:
         pass
 
 
@@ -22,6 +22,8 @@ class ExportFormat(str, Enum):
     dbt_sources = "dbt-sources"
     dbt_staging_sql = "dbt-staging-sql"
     odcs = "odcs"
+    odcs_v2 = "odcs_v2"
+    odcs_v3 = "odcs_v3"
     rdf = "rdf"
     avro = "avro"
     protobuf = "protobuf"
@@ -37,6 +39,7 @@ class ExportFormat(str, Enum):
     spark = "spark"
     sqlalchemy = "sqlalchemy"
     data_caterer = "data-caterer"
+    dcs = "dcs"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/export/exporter_factory.py

@@ -1,6 +1,7 @@
 import importlib
 import sys
-from datacontract.export.exporter import ExportFormat, Exporter
+
+from datacontract.export.exporter import Exporter, ExportFormat
 
 
 class ExporterFactory:
@@ -99,7 +100,15 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs, module_path="datacontract.export.odcs_converter", class_name="OdcsExporter"
+    name=ExportFormat.odcs_v2, module_path="datacontract.export.odcs_v2_exporter", class_name="OdcsV2Exporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.odcs_v3, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.odcs, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
 )
 
 exporter_factory.register_lazy_exporter(
@@ -109,7 +118,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.great_expectations,
     module_path="datacontract.export.great_expectations_converter",
-    class_name="GreateExpectationsExporter",
+    class_name="GreatExpectationsExporter",
 )
 
 exporter_factory.register_lazy_exporter(
@@ -155,3 +164,7 @@ exporter_factory.register_lazy_exporter(
     module_path="datacontract.export.sqlalchemy_converter",
     class_name="SQLAlchemyExporter",
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dcs, module_path="datacontract.export.dcs_exporter", class_name="DcsExporter"
+)

datacontract/export/go_converter.py

@@ -1,6 +1,7 @@
-import datacontract.model.data_contract_specification as spec
-from typing import List
 import re
+from typing import List
+
+import datacontract.model.data_contract_specification as spec
 from datacontract.export.exporter import Exporter