datacontract-cli 0.10.14__py3-none-any.whl → 0.10.15__py3-none-any.whl

datacontract/engines/fastjsonschema/check_jsonschema.py

@@ -1,31 +1,114 @@
 import json
 import logging
 import os
+import threading
+from typing import List, Optional
 
 import fastjsonschema
+from fastjsonschema import JsonSchemaValueException
 
 from datacontract.engines.fastjsonschema.s3.s3_read_files import yield_s3_files
 from datacontract.export.jsonschema_converter import to_jsonschema
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, Run
 
+# Thread-safe cache for primaryKey fields.
+_primary_key_cache = {}
+_cache_lock = threading.Lock()
 
-def validate_json_stream(model_name, validate, json_stream):
+
+def get_primary_key_field(schema: dict, model_name: str) -> Optional[str]:
+    # Check cache first.
+    with _cache_lock:
+        cached_value = _primary_key_cache.get(model_name)
+        if cached_value is not None:
+            return cached_value
+
+    # Find primaryKey field.
+    fields = schema.get("properties", {})
+    for field_name, attributes in fields.items():
+        if attributes.get("primaryKey", False):
+            # Cache the result before returning.
+            with _cache_lock:
+                _primary_key_cache[model_name] = field_name
+            return field_name
+
+    # Return None if no primary key was found.
+    return None
+
+
+def get_primary_key_value(schema: dict, model_name: str, json_object: dict) -> Optional[str]:
+    # Get the `primaryKey` field.
+    primary_key_field = get_primary_key_field(schema, model_name)
+    if not primary_key_field:
+        return None
+
+    # Return the value of the `primaryKey` field in the JSON object.
+    return json_object.get(primary_key_field)
+
+
+def process_exceptions(run, exceptions: List[DataContractException]):
+    if not exceptions:
+        return
+
+    # Define the maximum number of errors to process (can be adjusted by defining an ENV variable).
     try:
-        logging.info("Validating JSON")
-        for json_obj in json_stream:
+        error_limit = int(os.getenv("DATACONTRACT_MAX_ERRORS", 500))
+    except ValueError:
+        # Fallback to default if environment variable is invalid.
+        error_limit = 500
+
+    # Calculate the effective limit to avoid index out of range
+    limit = min(len(exceptions), error_limit)
+
+    # Add all exceptions up to the limit - 1 to `run.checks`.
+    DEFAULT_ERROR_MESSAGE = "An error occurred during validation phase. See the logs for more details."
+    run.checks.extend(
+        [
+            Check(
+                type=exception.type,
+                name=exception.name,
+                result=exception.result,
+                reason=exception.reason,
+                model=exception.model,
+                engine=exception.engine,
+                message=exception.message or DEFAULT_ERROR_MESSAGE,
+            )
+            for exception in exceptions[: limit - 1]
+        ]
+    )
+
+    # Raise the last exception within the limit.
+    last_exception = exceptions[limit - 1]
+    raise last_exception
+
+
+def validate_json_stream(
+    schema: dict, model_name: str, validate: callable, json_stream: list[dict]
+) -> List[DataContractException]:
+    logging.info(f"Validating JSON stream for model: '{model_name}'.")
+    exceptions: List[DataContractException] = []
+    for json_obj in json_stream:
+        try:
             validate(json_obj)
-        return True
-    except fastjsonschema.JsonSchemaValueException as e:
-        raise DataContractException(
-            type="schema",
-            name="Check that JSON has valid schema",
-            model=model_name,
-            reason=e.message,
-            engine="jsonschema",
-            original_exception=e,
-        )
+        except JsonSchemaValueException as e:
+            logging.warning(f"Validation failed for JSON object with type: '{model_name}'.")
+            primary_key_value = get_primary_key_value(schema, model_name, json_obj)
+            exceptions.append(
+                DataContractException(
+                    type="schema",
+                    name="Check that JSON has valid schema",
+                    result="failed",
+                    reason=f"{f'#{primary_key_value}: ' if primary_key_value is not None else ''}{e.message}",
+                    model=model_name,
+                    engine="jsonschema",
+                    message=e.message,
+                )
+            )
+    if not exceptions:
+        logging.info(f"All JSON objects in the stream passed validation for model: '{model_name}'.")
+    return exceptions
 
 
 def read_json_lines(file):
@@ -59,17 +142,22 @@ def read_json_file_content(file_content: str):
     yield json.loads(file_content)
 
 
-def process_json_file(run, model_name, validate, file, delimiter):
+def process_json_file(run, schema, model_name, validate, file, delimiter):
     if delimiter == "new_line":
         json_stream = read_json_lines(file)
     elif delimiter == "array":
         json_stream = read_json_array(file)
     else:
         json_stream = read_json_file(file)
-    validate_json_stream(model_name, validate, json_stream)
 
+    # Validate the JSON stream and collect exceptions.
+    exceptions = validate_json_stream(schema, model_name, validate, json_stream)
 
-def process_local_file(run, server, model_name, validate):
+    # Handle all errors from schema validation.
+    process_exceptions(run, exceptions)
+
+
+def process_local_file(run, server, schema, model_name, validate):
     path = server.path
     if "{model}" in path:
         path = path.format(model=model_name)
@@ -79,7 +167,7 @@ def process_local_file(run, server, model_name, validate):
     else:
         logging.info(f"Processing file {path}")
         with open(path, "r") as file:
-            process_json_file(run, model_name, validate, file, server.delimiter)
+            process_json_file(run, schema, model_name, validate, file, server.delimiter)
 
 
 def process_directory(run, path, server, model_name, validate):
@@ -94,7 +182,7 @@ def process_directory(run, path, server, model_name, validate):
     return success
 
 
-def process_s3_file(server, model_name, validate):
+def process_s3_file(run, server, schema, model_name, validate):
     s3_endpoint_url = server.endpointUrl
     s3_location = server.location
     if "{model}" in s3_location:
@@ -118,7 +206,11 @@ def process_s3_file(server, model_name, validate):
             engine="datacontract",
         )
 
-    return validate_json_stream(model_name, validate, json_stream)
+    # Validate the JSON stream and collect exceptions.
+    exceptions = validate_json_stream(schema, model_name, validate, json_stream)
+
+    # Handle all errors from schema validation.
+    process_exceptions(run, exceptions)
 
 
 def check_jsonschema(run: Run, data_contract: DataContractSpecification, server: Server):
@@ -155,9 +247,9 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
 
         # Process files based on server type
         if server.type == "local":
-            process_local_file(run, server, model_name, validate)
+            process_local_file(run, server, schema, model_name, validate)
         elif server.type == "s3":
-            process_s3_file(server, model_name, validate)
+            process_s3_file(run, server, schema, model_name, validate)
         elif server.type == "gcs":
             run.checks.append(
                 Check(

datacontract/engines/soda/check_soda_execute.py

@@ -12,7 +12,7 @@ from datacontract.engines.soda.connections.sqlserver import to_sqlserver_soda_co
 from datacontract.engines.soda.connections.trino import to_trino_soda_configuration
 from datacontract.export.sodacl_converter import to_sodacl_yaml
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
-from datacontract.model.run import Run, Check, Log
+from datacontract.model.run import Check, Log, Run
 
 
 def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark, tmp_dir):
@@ -176,9 +176,11 @@ def update_reason(check, c):
         if block["title"] == "Diagnostics":
             # Extract and print the 'text' value
             diagnostics_text = block["text"]
-            print(diagnostics_text)
+            # print(diagnostics_text)
             diagnostics_text_split = diagnostics_text.split(":icon-fail: ")
             if len(diagnostics_text_split) > 1:
                 check.reason = diagnostics_text_split[1].strip()
-                print(check.reason)
+                # print(check.reason)
             break  # Exit the loop once the desired block is found
+    if c["diagnostics"]["fail"] is not None:
+        check.reason = f"Got: {c['diagnostics']['value']} Expected: {c['diagnostics']['fail']}"

datacontract/engines/soda/connections/duckdb.py

@@ -1,6 +1,7 @@
 import os
 
 import duckdb
+
 from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type
 from datacontract.model.run import Run
 

datacontract/engines/soda/connections/kafka.py

@@ -2,7 +2,7 @@ import logging
 import os
 
 from datacontract.export.avro_converter import to_avro_schema_json
-from datacontract.model.data_contract_specification import DataContractSpecification, Server, Field
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Server
 from datacontract.model.exceptions import DataContractException
 
 
@@ -69,8 +69,8 @@ def read_kafka_topic(spark, data_contract: DataContractSpecification, server: Se
 
 def process_avro_format(df, model_name, model):
     try:
-        from pyspark.sql.functions import col, expr
         from pyspark.sql.avro.functions import from_avro
+        from pyspark.sql.functions import col, expr
     except ImportError as e:
         raise DataContractException(
             type="schema",
@@ -167,21 +167,21 @@ def to_struct_type(fields):
 def to_struct_field(field_name: str, field: Field):
     try:
         from pyspark.sql.types import (
-            StructType,
-            StructField,
-            StringType,
+            ArrayType,
+            BinaryType,
+            BooleanType,
+            DataType,
+            DateType,
             DecimalType,
             DoubleType,
             IntegerType,
             LongType,
-            BooleanType,
-            TimestampType,
-            TimestampNTZType,
-            DateType,
-            BinaryType,
-            ArrayType,
             NullType,
-            DataType,
+            StringType,
+            StructField,
+            StructType,
+            TimestampNTZType,
+            TimestampType,
         )
     except ImportError as e:
         raise DataContractException(

datacontract/export/avro_idl_converter.py

@@ -3,12 +3,11 @@ from dataclasses import dataclass
 from enum import Enum
 from io import StringIO
 
+from datacontract.export.exporter import Exporter
 from datacontract.lint.resolve import inline_definitions_into_data_contract
 from datacontract.model.data_contract_specification import DataContractSpecification, Field
 from datacontract.model.exceptions import DataContractException
 
-from datacontract.export.exporter import Exporter
-
 
 class AvroPrimitiveType(Enum):
     int = "int"

datacontract/export/bigquery_converter.py

@@ -2,10 +2,9 @@ import json
 import logging
 from typing import Dict, List
 
-from datacontract.model.data_contract_specification import Model, Field, Server
-from datacontract.model.exceptions import DataContractException
-
 from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.model.data_contract_specification import Field, Model, Server
+from datacontract.model.exceptions import DataContractException
 
 
 class BigQueryExporter(Exporter):
@@ -109,6 +108,8 @@ def map_type_to_bigquery(field: Field) -> str:
         return "NUMERIC"
     elif field_type.lower() == "double":
         return "BIGNUMERIC"
+    elif field_type.lower() in ["object", "record"] and not field.fields:
+        return "JSON"
     elif field_type.lower() in ["object", "record", "array"]:
         return "RECORD"
     elif field_type.lower() == "struct":

datacontract/export/data_caterer_converter.py

@@ -3,7 +3,7 @@ from typing import Dict
 import yaml
 
 from datacontract.export.exporter import Exporter
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field, Server
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server
 
 
 class DataCatererExporter(Exporter):

datacontract/export/dbml_converter.py

@@ -3,13 +3,11 @@ from importlib.metadata import version
 from typing import Tuple
 
 import pytz
-from datacontract.model.exceptions import DataContractException
 
 import datacontract.model.data_contract_specification as spec
-from datacontract.export.sql_type_converter import convert_to_sql_type
-
-
 from datacontract.export.exporter import Exporter
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.exceptions import DataContractException
 
 
 class DbmlExporter(Exporter):

datacontract/export/dbt_converter.py

@@ -2,10 +2,9 @@ from typing import Dict
 
 import yaml
 
-from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
-
 from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
 
 class DbtExporter(Exporter):

datacontract/export/exporter.py

@@ -1,6 +1,6 @@
+import typing
 from abc import ABC, abstractmethod
 from enum import Enum
-import typing
 
 from datacontract.model.data_contract_specification import DataContractSpecification
 

datacontract/export/exporter_factory.py

@@ -1,6 +1,7 @@
 import importlib
 import sys
-from datacontract.export.exporter import ExportFormat, Exporter
+
+from datacontract.export.exporter import Exporter, ExportFormat
 
 
 class ExporterFactory:
@@ -117,7 +118,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.great_expectations,
     module_path="datacontract.export.great_expectations_converter",
-    class_name="GreateExpectationsExporter",
+    class_name="GreatExpectationsExporter",
 )
 
 exporter_factory.register_lazy_exporter(

datacontract/export/go_converter.py

@@ -1,6 +1,7 @@
-import datacontract.model.data_contract_specification as spec
-from typing import List
 import re
+from typing import List
+
+import datacontract.model.data_contract_specification as spec
 from datacontract.export.exporter import Exporter