datacontract-cli 0.9.8__py3-none-any.whl → 0.10.0__py3-none-any.whl

datacontract/export/sodacl_converter.py

@@ -5,7 +5,9 @@ from datacontract.model.data_contract_specification import \
     DataContractSpecification
 
 
-def to_sodacl_yaml(data_contract_spec: DataContractSpecification, server_type: str = None, check_types: bool = True) -> str:
+def to_sodacl_yaml(
+    data_contract_spec: DataContractSpecification, server_type: str = None, check_types: bool = True
+) -> str:
     try:
         sodacl = {}
         for model_key, model_value in data_contract_spec.models.items():
@@ -33,6 +35,26 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
             checks.append(check_field_required(field_name, quote_field_name))
         if field.unique:
             checks.append(check_field_unique(field_name, quote_field_name))
+        if field.minLength is not None:
+            checks.append(check_field_min_length(field_name, field.minLength))
+        if field.maxLength is not None:
+            checks.append(check_field_max_length(field_name, field.maxLength))
+        if field.minimum is not None:
+            checks.append(check_field_minimum(field_name, field.minimum))
+        if field.maximum is not None:
+            checks.append(check_field_maximum(field_name, field.maximum))
+        if field.exclusiveMinimum is not None:
+            checks.append(check_field_minimum(field_name, field.exclusiveMinimum))
+            checks.append(check_field_not_equal(field_name, field.exclusiveMinimum))
+        if field.exclusiveMaximum is not None:
+            checks.append(check_field_maximum(field_name, field.exclusiveMaximum))
+            checks.append(check_field_not_equal(field_name, field.exclusiveMaximum))
+        if field.pattern is not None:
+            checks.append(check_field_regex(field_name, field.pattern))
+        if field.enum is not None and len(field.enum) > 0:
+            checks.append(check_field_enum(field_name, field.enum))
+        # TODO references: str = None
+        # TODO format
 
     return f"checks for {model_key}", checks
 
@@ -59,18 +81,93 @@ def check_field_type(field_name: str, type: str):
 
 def check_field_required(field_name: str, quote_field_name: bool = False):
     if quote_field_name:
-        field_name = f"\"{field_name}\""
+        field_name = f'"{field_name}"'
 
-    return {
-        f"missing_count({field_name}) = 0": {"name": f"Check that required field {field_name} has no null values"}}
+    return {f"missing_count({field_name}) = 0": {"name": f"Check that required field {field_name} has no null values"}}
 
 
 def check_field_unique(field_name, quote_field_name: bool = False):
     if quote_field_name:
-        field_name = f"\"{field_name}\""
+        field_name = f'"{field_name}"'
+    return {
+        f"duplicate_count({field_name}) = 0": {"name": f"Check that unique field {field_name} has no duplicate values"}
+    }
+
+
+def check_field_min_length(field_name, min_length, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a min length of {min}",
+            "valid min length": min_length,
+        }
+    }
+
+
+def check_field_max_length(field_name, max_length, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
     return {
-        f"duplicate_count({field_name}) = 0": {
-            "name": f"Check that unique field {field_name} has no duplicate values"}
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a max length of {max_length}",
+            "valid max length": max_length,
+        }
+    }
+
+
+def check_field_minimum(field_name, minimum, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a minimum of {min}",
+            "valid min": minimum,
+        }
+    }
+
+
+def check_field_maximum(field_name, maximum, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a maximum of {maximum}",
+            "valid max": maximum,
+        }
+    }
+
+
+def check_field_not_equal(field_name, value, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} is not equal to {value}",
+            "invalid values": [value],
+        }
+    }
+
+
+def check_field_enum(field_name, enum, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} only contains enum values {enum}",
+            "valid values": enum,
+        }
+    }
+
+
+def check_field_regex(field_name, pattern, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} matches regex pattern {pattern}",
+            "valid regex": pattern,
+        }
     }
 
 

datacontract/export/sql_converter.py

@@ -85,9 +85,20 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
             result += " not null"
         if field.primary:
             result += " primary key"
+        if server_type == "databricks" and field.description is not None:
+            result += f' COMMENT "{_escape(field.description)}"'
         if current_field_index < fields:
             result += ","
         result += "\n"
         current_field_index += 1
-    result += ");\n"
+    result += ")"
+    if server_type == "databricks" and model.description is not None:
+        result += f' COMMENT "{_escape(model.description)}"'
+    result += ";\n"
     return result
+
+
+def _escape(text: str | None) -> str | None:
+    if text is None:
+        return None
+    return text.replace('"', '\\"')

datacontract/imports/avro_importer.py

@@ -26,7 +26,6 @@ def import_avro(data_contract_specification: DataContractSpecification, source:
     fields = import_record_fields(avro_schema.fields)
 
     data_contract_specification.models[avro_schema.name] = Model(
-        type="table",
         fields=fields,
     )
 
@@ -42,10 +41,11 @@ def import_avro(data_contract_specification: DataContractSpecification, source:
 def import_record_fields(record_fields):
     imported_fields = {}
     for field in record_fields:
-
         imported_fields[field.name] = Field()
         imported_fields[field.name].required = True
         imported_fields[field.name].description = field.doc
+        for prop in field.other_props:
+            imported_fields[field.name].__setattr__(prop, field.other_props[prop])
 
         if field.type.type == "record":
             imported_fields[field.name].type = "object"
@@ -53,12 +53,36 @@ def import_record_fields(record_fields):
             imported_fields[field.name].fields = import_record_fields(field.type.fields)
         elif field.type.type == "union":
             imported_fields[field.name].required = False
-            imported_fields[field.name].type = import_type_of_optional_field(field)
-        else: # primitive type
+            type = import_type_of_optional_field(field)
+            imported_fields[field.name].type = type
+            if type == "record":
+                imported_fields[field.name].fields = import_record_fields(get_record_from_union_field(field).fields)
+        elif field.type.type == "array":
+            imported_fields[field.name].type = "array"
+            imported_fields[field.name].items = import_avro_array_items(field.type)
+        else:  # primitive type
             imported_fields[field.name].type = map_type_from_avro(field.type.type)
+
     return imported_fields
 
 
+def import_avro_array_items(array_schema):
+    items = Field()
+    for prop in array_schema.other_props:
+        items.__setattr__(prop, array_schema.other_props[prop])
+
+    if array_schema.items.type == "record":
+        items.type = "object"
+        items.fields = import_record_fields(array_schema.items.fields)
+    elif array_schema.items.type == "array":
+        items.type = "array"
+        items.items = import_avro_array_items(array_schema.items)
+    else:  # primitive type
+        items.type = map_type_from_avro(array_schema.items.type)
+
+    return items
+
+
 def import_type_of_optional_field(field):
     for field_type in field.type.schemas:
         if field_type.type != "null":
@@ -72,6 +96,13 @@ def import_type_of_optional_field(field):
     )
 
 
+def get_record_from_union_field(field):
+    for field_type in field.type.schemas:
+        if field_type.type == "record":
+            return field_type
+    return None
+
+
 def map_type_from_avro(avro_type_str: str):
     # TODO: ambiguous mapping in the export
     if avro_type_str == "null":
@@ -88,14 +119,8 @@ def map_type_from_avro(avro_type_str: str):
         return "long"
     elif avro_type_str == "boolean":
         return "boolean"
-    elif avro_type_str == "array":
-        raise DataContractException(
-            type="schema",
-            result="failed",
-            name="Map avro type to data contract type",
-            reason="Array type not supported",
-            engine="datacontract",
-        )
+    elif avro_type_str == "record":
+        return "record"
     else:
         raise DataContractException(
             type="schema",

datacontract/integration/publish_datamesh_manager.py

@@ -1,4 +1,3 @@
-import logging
 import os
 
 import requests
@@ -29,6 +28,6 @@ def publish_datamesh_manager(run: Run, publish_url: str):
         if response.status_code != 200:
             run.log_error(f"Error publishing test results to Data Mesh Manager: {response.text}")
             return
-        logging.info("Published test results to %s", url)
+        run.log_info(f"Published test results to {url}")
     except Exception as e:
-        logging.error(f"Failed publishing test results. Error: {str(e)}")
+        run.log_error(f"Failed publishing test results. Error: {str(e)}")

datacontract/lint/resolve.py

@@ -1,4 +1,5 @@
 import logging
+import os
 
 import fastjsonschema
 import yaml
@@ -8,7 +9,7 @@ from datacontract.lint.files import read_file
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
 from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Definition
+    DataContractSpecification, Definition, Quality
 from datacontract.model.exceptions import DataContractException
 
 
@@ -36,13 +37,13 @@ def resolve_data_contract(
 
 
 def resolve_data_contract_from_location(
-    location, schema_location: str = None, inline_definitions: bool = False
+    location, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = True
 ) -> DataContractSpecification:
     if location.startswith("http://") or location.startswith("https://"):
         data_contract_str = fetch_resource(location)
     else:
         data_contract_str = read_file(location)
-    return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions)
+    return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, include_quality)
 
 
 def inline_definitions_into_data_contract(spec: DataContractSpecification):
@@ -52,7 +53,7 @@ def inline_definitions_into_data_contract(spec: DataContractSpecification):
             if not field.ref and not field.ref_obj:
                 continue
 
-            definition = resolve_ref(field.ref, spec.definitions)
+            definition = resolve_definition_ref(field.ref, spec.definitions)
             field.ref_obj = definition
 
             for field_name in field.model_fields.keys():
@@ -60,7 +61,7 @@ def inline_definitions_into_data_contract(spec: DataContractSpecification):
                     setattr(field, field_name, getattr(definition, field_name))
 
 
-def resolve_ref(ref, definitions) -> Definition:
+def resolve_definition_ref(ref, definitions) -> Definition:
     if ref.startswith("http://") or ref.startswith("https://"):
         definition_str = fetch_resource(ref)
         definition_dict = to_yaml(definition_str)
@@ -79,8 +80,44 @@ def resolve_ref(ref, definitions) -> Definition:
         )
 
 
+def resolve_quality_ref(quality: Quality):
+    """
+    Return the content of a ref file path
+    @param quality data contract quality specification
+    """
+    if isinstance(quality.specification, dict):
+        specification = quality.specification
+        if quality.type == "great-expectations":
+            for model, model_quality in specification.items():
+                specification[model] = get_quality_ref_file(model_quality)
+        else:
+            if "$ref" in specification:
+                quality.specification = get_quality_ref_file(specification)
+
+
+def get_quality_ref_file(quality_spec: str | object) -> str | object:
+    """
+    Get the file associated with a quality reference
+    @param quality_spec quality specification
+    @returns: the content of the quality file
+    """
+    if isinstance(quality_spec, dict) and "$ref" in quality_spec:
+        ref = quality_spec["$ref"]
+        if not os.path.exists(ref):
+            raise DataContractException(
+                type="export",
+                result="failed",
+                name="Check that data contract quality is valid",
+                reason=f"Cannot resolve reference {ref}",
+                engine="datacontract",
+            )
+        with open(ref, "r") as file:
+            quality_spec = file.read()
+    return quality_spec
+
+
 def resolve_data_contract_from_str(
-    data_contract_str, schema_location: str = None, inline_definitions: bool = False
+    data_contract_str, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = False
 ) -> DataContractSpecification:
     data_contract_yaml_dict = to_yaml(data_contract_str)
     validate(data_contract_yaml_dict, schema_location)
@@ -89,6 +126,8 @@ def resolve_data_contract_from_str(
 
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
+    if spec.quality and include_quality:
+        resolve_quality_ref(spec.quality)
 
     return spec
 

datacontract/model/run.py

@@ -15,6 +15,7 @@ class Check(BaseModel):
     model: Optional[str] = None
     field: Optional[str] = None
     details: Optional[str] = None
+    diagnostics: Optional[dict] = None
 
 
 class Log(BaseModel):
@@ -69,7 +70,7 @@ class Run(BaseModel):
         self.logs.append(Log(level="ERROR", message=message, timestamp=datetime.now(timezone.utc)))
 
     def pretty(self):
-        return self.model_dump_json()
+        return self.model_dump_json(indent=2)
 
     @staticmethod
     def create_run():