datacontract-cli 0.10.34__py3-none-any.whl → 0.10.36__py3-none-any.whl

datacontract/engines/fastjsonschema/check_jsonschema.py

@@ -1,8 +1,9 @@
+import glob
 import json
 import logging
 import os
 import threading
-from typing import List, Optional
+from typing import Any, Callable, Generator, List, Optional
 
 import fastjsonschema
 from fastjsonschema import JsonSchemaValueException
@@ -85,7 +86,7 @@ def process_exceptions(run, exceptions: List[DataContractException]):
 
 
 def validate_json_stream(
-    schema: dict, model_name: str, validate: callable, json_stream: list[dict]
+    schema: dict, model_name: str, validate: Callable, json_stream: Generator[Any, Any, None]
 ) -> List[DataContractException]:
     logging.info(f"Validating JSON stream for model: '{model_name}'.")
     exceptions: List[DataContractException] = []
@@ -99,7 +100,7 @@ def validate_json_stream(
                 DataContractException(
                     type="schema",
                     name="Check that JSON has valid schema",
-                    result="failed",
+                    result=ResultEnum.failed,
                     reason=f"{f'#{primary_key_value}: ' if primary_key_value is not None else ''}{e.message}",
                     model=model_name,
                     engine="jsonschema",
@@ -170,24 +171,33 @@ def process_local_file(run, server, schema, model_name, validate):
     if "{model}" in path:
         path = path.format(model=model_name)
 
+    all_files = []
     if os.path.isdir(path):
-        return process_directory(run, path, server, model_name, validate)
+        # Fetch all JSONs in the directory
+        for root, _, files in os.walk(path):
+            for file in files:
+                if file.endswith(".json"):
+                    all_files.append(os.path.join(root, file))
     else:
-        logging.info(f"Processing file {path}")
-        with open(path, "r") as file:
-            process_json_file(run, schema, model_name, validate, file, server.delimiter)
+        # Use glob to fetch all JSONs
+        for file_path in glob.glob(path, recursive=True):
+            if os.path.isfile(file_path):
+                if file_path.endswith(".json"):
+                    all_files.append(file_path)
 
+    if not all_files:
+        raise DataContractException(
+            type="schema",
+            name="Check that JSON has valid schema",
+            result=ResultEnum.warning,
+            reason=f"No files found in '{path}'.",
+            engine="datacontract",
+        )
 
-def process_directory(run, path, server, model_name, validate):
-    success = True
-    for filename in os.listdir(path):
-        if filename.endswith(".json"):  # or make this a parameter
-            file_path = os.path.join(path, filename)
-            with open(file_path, "r") as file:
-                if not process_json_file(run, model_name, validate, file, server.delimiter):
-                    success = False
-                    break
-    return success
+    for file in all_files:
+        logging.info(f"Processing file: {file}")
+        with open(file, "r") as f:
+            process_json_file(run, schema, model_name, validate, f, server.delimiter)
 
 
 def process_s3_file(run, server, schema, model_name, validate):
@@ -209,7 +219,7 @@ def process_s3_file(run, server, schema, model_name, validate):
         raise DataContractException(
             type="schema",
             name="Check that JSON has valid schema",
-            result="warning",
+            result=ResultEnum.warning,
             reason=f"Cannot find any file in {s3_location}",
             engine="datacontract",
         )
@@ -230,7 +240,7 @@ def check_jsonschema(run: Run, data_contract: DataContractSpecification, server:
             Check(
                 type="schema",
                 name="Check that JSON has valid schema",
-                result="warning",
+                result=ResultEnum.warning,
                 reason="Server format is not 'json'. Skip validating jsonschema.",
                 engine="jsonschema",
             )

datacontract/export/dbt_converter.py

@@ -115,9 +115,28 @@ def _to_dbt_model(
         dbt_model["config"]["contract"] = {"enforced": True}
     if model_value.description is not None:
         dbt_model["description"] = model_value.description.strip().replace("\n", " ")
-    columns = _to_columns(data_contract_spec, model_value.fields, _supports_constraints(model_type), adapter_type)
+
+    # Handle model-level primaryKey (before columns for better YAML ordering)
+    primary_key_columns = []
+    if hasattr(model_value, "primaryKey") and model_value.primaryKey:
+        if isinstance(model_value.primaryKey, list) and len(model_value.primaryKey) > 1:
+            # Multiple columns: use dbt_utils.unique_combination_of_columns
+            dbt_model["data_tests"] = [
+                {"dbt_utils.unique_combination_of_columns": {"combination_of_columns": model_value.primaryKey}}
+            ]
+        elif isinstance(model_value.primaryKey, list) and len(model_value.primaryKey) == 1:
+            # Single column: handle at column level (pass to _to_columns)
+            primary_key_columns = model_value.primaryKey
+        elif isinstance(model_value.primaryKey, str):
+            # Single column as string: handle at column level
+            primary_key_columns = [model_value.primaryKey]
+
+    columns = _to_columns(
+        data_contract_spec, model_value.fields, _supports_constraints(model_type), adapter_type, primary_key_columns
+    )
     if columns:
         dbt_model["columns"] = columns
+
     return dbt_model
 
 
@@ -143,10 +162,13 @@ def _to_columns(
     fields: Dict[str, Field],
     supports_constraints: bool,
     adapter_type: Optional[str],
+    primary_key_columns: Optional[list] = None,
 ) -> list:
     columns = []
+    primary_key_columns = primary_key_columns or []
     for field_name, field in fields.items():
-        column = _to_column(data_contract_spec, field_name, field, supports_constraints, adapter_type)
+        is_primary_key = field_name in primary_key_columns
+        column = _to_column(data_contract_spec, field_name, field, supports_constraints, adapter_type, is_primary_key)
         columns.append(column)
     return columns
 
@@ -164,6 +186,7 @@ def _to_column(
     field: Field,
     supports_constraints: bool,
     adapter_type: Optional[str],
+    is_primary_key: bool = False,
 ) -> dict:
     column = {"name": field_name}
     adapter_type = adapter_type or "snowflake"
@@ -178,12 +201,15 @@ def _to_column(
         )
     if field.description is not None:
         column["description"] = field.description.strip().replace("\n", " ")
-    if field.required:
+    # Handle required/not_null constraint
+    if field.required or is_primary_key:
         if supports_constraints:
             column.setdefault("constraints", []).append({"type": "not_null"})
         else:
             column["data_tests"].append("not_null")
-    if field.unique:
+
+    # Handle unique constraint
+    if field.unique or is_primary_key:
         if supports_constraints:
             column.setdefault("constraints", []).append({"type": "unique"})
         else:

datacontract/export/dqx_converter.py

@@ -0,0 +1,126 @@
+from typing import Any, Dict, List, Union
+
+import yaml
+
+from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Quality
+
+
+class DqxKeys:
+    CHECK = "check"
+    ARGUMENTS = "arguments"
+    SPECIFICATION = "specification"
+    COL_NAME = "column"
+    COL_NAMES = "for_each_column"
+    COLUMNS = "columns"
+    FUNCTION = "function"
+
+
+class DqxExporter(Exporter):
+    """Exporter implementation for converting data contracts to DQX YAML file."""
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model: Model,
+        server: str,
+        sql_server_type: str,
+        export_args: Dict[str, Any],
+    ) -> str:
+        """Exports a data contract to DQX format."""
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_dqx_yaml(model_value)
+
+
+def to_dqx_yaml(model_value: Model) -> str:
+    """
+    Converts the data contract's quality checks to DQX YAML format.
+
+    Args:
+        model_value (Model): The data contract to convert.
+
+    Returns:
+        str: YAML representation of the data contract's quality checks.
+    """
+    extracted_rules = extract_quality_rules(model_value)
+    return yaml.dump(extracted_rules, sort_keys=False, allow_unicode=True, default_flow_style=False)
+
+
+def process_quality_rule(rule: Quality, column_name: str) -> Dict[str, Any]:
+    """
+    Processes a single quality rule by injecting the column path into its arguments if absent.
+
+    Args:
+        rule (Quality): The quality rule to process.
+        column_name (str): The full path to the current column.
+
+    Returns:
+        dict: The processed quality rule specification.
+    """
+    rule_data = rule.model_extra
+    specification = rule_data[DqxKeys.SPECIFICATION]
+    check = specification[DqxKeys.CHECK]
+
+    if column_name:
+        arguments = check.setdefault(DqxKeys.ARGUMENTS, {})
+
+        if (
+            DqxKeys.COL_NAME not in arguments
+            and DqxKeys.COL_NAMES not in arguments
+            and DqxKeys.COLUMNS not in arguments
+        ):
+            if check[DqxKeys.FUNCTION] not in ("is_unique", "foreign_key"):
+                arguments[DqxKeys.COL_NAME] = column_name
+            else:
+                arguments[DqxKeys.COLUMNS] = [column_name]
+
+    return specification
+
+
+def extract_quality_rules(data: Union[Model, Field, Quality], column_path: str = "") -> List[Dict[str, Any]]:
+    """
+    Recursively extracts all quality rules from a data contract structure.
+
+    Args:
+        data (Union[Model, Field, Quality]): The data contract model, field, or quality rule.
+        column_path (str, optional): The current path in the schema hierarchy. Defaults to "".
+
+    Returns:
+        List[Dict[str, Any]]: A list of quality rule specifications.
+    """
+    quality_rules = []
+
+    if isinstance(data, Quality):
+        return [process_quality_rule(data, column_path)]
+
+    if isinstance(data, (Model, Field)):
+        for key, field in data.fields.items():
+            current_path = build_column_path(column_path, key)
+
+            if field.fields:
+                # Field is a struct-like object, recurse deeper
+                quality_rules.extend(extract_quality_rules(field, current_path))
+            else:
+                # Process quality rules at leaf fields
+                for rule in field.quality:
+                    quality_rules.append(process_quality_rule(rule, current_path))
+
+        # Process any quality rules attached directly to this level
+        for rule in data.quality:
+            quality_rules.append(process_quality_rule(rule, column_path))
+
+    return quality_rules
+
+
+def build_column_path(current_path: str, key: str) -> str:
+    """
+    Builds the full column path by concatenating parent path with current key.
+
+    Args:
+        current_path (str): The current path prefix.
+        key (str): The current field's key.
+
+    Returns:
+        str: The full path.
+    """
+    return f"{current_path}.{key}" if current_path else key

datacontract/export/excel_exporter.py

@@ -283,7 +283,7 @@ def fill_single_property_template(
     sheet: Worksheet, row_index: int, prefix: str, property: SchemaProperty, header_map: dict
 ) -> int:
     """Fill a single property row using the template's column structure"""
-    property_name = f"{prefix}.{property.name}" if prefix else property.name
+    property_name = f"{prefix}{'.' + property.name if property.name else ''}" if prefix else property.name
 
     # Helper function to set cell value by header name
     def set_by_header(header_name: str, value: Any):
@@ -307,7 +307,7 @@ def fill_single_property_template(
     set_by_header("Classification", property.classification)
     set_by_header("Tags", ",".join(property.tags) if property.tags else "")
     set_by_header(
-        "Example(s)", ",".join(property.examples) if property.examples else ""
+        "Example(s)", ",".join(map(str, property.examples)) if property.examples else ""
     )  # Note: using "Example(s)" as in template
     set_by_header("Encrypted Name", property.encryptedName)
     set_by_header(
@@ -404,7 +404,7 @@ def fill_properties_quality(
         if not property.name:
             continue
 
-        full_property_name = f"{prefix}.{property.name}" if prefix else property.name
+        full_property_name = f"{prefix}{'.' + property.name if property.name else ''}" if prefix else property.name
 
         # Add quality attributes for this property
         if property.quality:

datacontract/export/exporter.py

@@ -46,6 +46,7 @@ class ExportFormat(str, Enum):
     iceberg = "iceberg"
     custom = "custom"
     excel = "excel"
+    dqx = "dqx"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/export/exporter_factory.py

@@ -197,6 +197,12 @@ exporter_factory.register_lazy_exporter(
     class_name="MarkdownExporter",
 )
 
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dqx,
+    module_path="datacontract.export.dqx_converter",
+    class_name="DqxExporter",
+)
+
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
 )

datacontract/export/markdown_converter.py

@@ -82,7 +82,7 @@ def obj_attributes_to_markdown(obj: BaseModel, excluded_fields: set = set(), is_
         if value
     ]
     description = f"*{description_to_markdown(description_value)}*"
-    extra = [extra_to_markdown(obj)] if obj.model_extra else []
+    extra = [extra_to_markdown(obj, is_in_table_cell)] if obj.model_extra else []
     return newline_char.join([description] + attributes + extra)
 
 
@@ -293,26 +293,45 @@ def dict_to_markdown(dictionary: Dict[str, str]) -> str:
     return "\n".join(markdown_parts) + "\n"
 
 
-def extra_to_markdown(obj: BaseModel) -> str:
+def extra_to_markdown(obj: BaseModel, is_in_table_cell: bool = False) -> str:
     """
     Convert the extra attributes of a data contract to Markdown format.
     Args:
         obj (BaseModel): The data contract object containing extra attributes.
+        is_in_table_cell (bool): Whether the extra attributes are in a table cell.
     Returns:
         str: A Markdown formatted string representing the extra attributes of the data contract.
     """
-    markdown_part = ""
     extra = obj.model_extra
-    if extra:
-        for key_extra, value_extra in extra.items():
-            markdown_part += f"\n### {key_extra.capitalize()}\n"
-            if isinstance(value_extra, list) and len(value_extra):
-                if isinstance(value_extra[0], dict):
-                    markdown_part += array_of_dict_to_markdown(value_extra)
-                elif isinstance(value_extra[0], str):
-                    markdown_part += array_to_markdown(value_extra)
-            elif isinstance(value_extra, dict):
-                markdown_part += dict_to_markdown(value_extra)
-            else:
-                markdown_part += f"{str(value_extra)}\n"
-    return markdown_part
+
+    if not extra:
+        return ""
+
+    bullet_char = "•"
+    value_line_ending = "" if is_in_table_cell else "\n"
+    row_suffix = "<br>" if is_in_table_cell else ""
+
+    def render_header(key: str) -> str:
+        return f"{bullet_char} **{key}:** " if is_in_table_cell else f"\n### {key.capitalize()}\n"
+
+    parts: list[str] = []
+    for key_extra, value_extra in extra.items():
+        if not value_extra:
+            continue
+
+        parts.append(render_header(key_extra))
+
+        if isinstance(value_extra, list) and len(value_extra):
+            if isinstance(value_extra[0], dict):
+                parts.append(array_of_dict_to_markdown(value_extra))
+            elif isinstance(value_extra[0], str):
+                parts.append(array_to_markdown(value_extra))
+        elif isinstance(value_extra, dict):
+            parts.append(dict_to_markdown(value_extra))
+        else:
+            parts.append(f"{str(value_extra)}{value_line_ending}")
+
+        if row_suffix:
+            parts.append(row_suffix)
+
+    return "".join(parts)

datacontract/export/mermaid_exporter.py

@@ -27,31 +27,33 @@ def dcs_to_mermaid(data_contract_spec: DataContractSpecification) -> str | None:
         mmd_references = []
 
         for model_name, model in data_contract_spec.models.items():
+            clean_model = _sanitize_name(model_name)
             entity_block = ""
 
             for field_name, field in model.fields.items():
                 clean_name = _sanitize_name(field_name)
-                indicators = ""
+                field_type = field.type or "unknown"
 
-                if field.primaryKey or (field.unique and field.required):
-                    indicators += "🔑"
-                if field.references:
-                    indicators += "⌘"
+                is_pk = bool(field.primaryKey or (field.unique and field.required))
+                is_fk = bool(field.references)
 
-                field_type = field.type or "unknown"
-                entity_block += f"\t{clean_name}{indicators} {field_type}\n"
+                entity_block += _field_line(clean_name, field_type, pk=is_pk, uk=bool(field.unique), fk=is_fk)
 
                 if field.references:
-                    referenced_model = field.references.split(".")[0] if "." in field.references else ""
+                    references = field.references.replace(".", "·")
+                    parts = references.split("·")
+                    referenced_model = _sanitize_name(parts[0]) if len(parts) > 0 else ""
+                    referenced_field = _sanitize_name(parts[1]) if len(parts) > 1 else ""
                     if referenced_model:
-                        mmd_references.append(f'"📑{referenced_model}"' + "}o--{ ||" + f'"📑{model_name}"')
+                        label = referenced_field or clean_name
+                        mmd_references.append(f'"**{referenced_model}**" ||--o{{ "**{clean_model}**" : {label}')
 
-            mmd_entity += f'\t"**{model_name}**"' + "{\n" + entity_block + "}\n"
+            mmd_entity += f'\t"**{clean_model}**" {{\n{entity_block}}}\n'
 
         if mmd_references:
             mmd_entity += "\n" + "\n".join(mmd_references)
 
-        return f"{mmd_entity}\n"
+        return mmd_entity + "\n"
 
     except Exception as e:
         print(f"Error generating DCS mermaid diagram: {e}")
@@ -95,3 +97,14 @@ def odcs_to_mermaid(data_contract_spec: OpenDataContractStandard) -> str | None:
 
 def _sanitize_name(name: str) -> str:
     return name.replace("#", "Nb").replace(" ", "_").replace("/", "by")
+
+
+def _field_line(name: str, field_type: str, pk: bool = False, uk: bool = False, fk: bool = False) -> str:
+    indicators = ""
+    if pk:
+        indicators += "🔑"
+    if uk:
+        indicators += "🔒"
+    if fk:
+        indicators += "⌘"
+    return f"\t{name}{indicators} {field_type}\n"

datacontract/export/rdf_converter.py

@@ -57,8 +57,8 @@ def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
     else:
         g = Graph(base=Namespace(""))
 
-    dc = Namespace("https://datacontract.com/DataContractSpecification/1.2.0/")
-    dcx = Namespace("https://datacontract.com/DataContractSpecification/1.2.0/Extension/")
+    dc = Namespace("https://datacontract.com/DataContractSpecification/1.2.1/")
+    dcx = Namespace("https://datacontract.com/DataContractSpecification/1.2.1/Extension/")
 
     g.bind("dc", dc)
     g.bind("dcx", dcx)

datacontract/export/spark_converter.py

@@ -1,3 +1,5 @@
+import json
+
 from pyspark.sql import types
 
 from datacontract.export.exporter import Exporter
@@ -104,7 +106,8 @@ def to_struct_field(field: Field, field_name: str) -> types.StructField:
         types.StructField: The corresponding Spark StructField.
     """
     data_type = to_spark_data_type(field)
-    return types.StructField(name=field_name, dataType=data_type, nullable=not field.required)
+    metadata = to_spark_metadata(field)
+    return types.StructField(name=field_name, dataType=data_type, nullable=not field.required, metadata=metadata)
 
 
 def to_spark_data_type(field: Field) -> types.DataType:
@@ -152,7 +155,25 @@ def to_spark_data_type(field: Field) -> types.DataType:
         return types.DateType()
     if field_type == "bytes":
         return types.BinaryType()
-    return types.StringType() # default if no condition is met
+    return types.StringType()  # default if no condition is met
+
+
+def to_spark_metadata(field: Field) -> dict[str, str]:
+    """
+    Convert a field to a Spark metadata dictonary.
+
+    Args:
+        field (Field): The field to convert.
+
+    Returns:
+        dict: dictionary that can be supplied to Spark as metadata for a StructField
+    """
+
+    metadata = {}
+    if field.description:
+        metadata["comment"] = field.description
+
+    return metadata
 
 
 def print_schema(dtype: types.DataType) -> str:
@@ -192,7 +213,11 @@ def print_schema(dtype: types.DataType) -> str:
         name = f'"{column.name}"'
         data_type = indent(print_schema(column.dataType), 1)
         nullable = indent(f"{column.nullable}", 1)
-        return f"StructField({name},\n{data_type},\n{nullable}\n)"
+        if column.metadata:
+            metadata = indent(f"{json.dumps(column.metadata)}", 1)
+            return f"StructField({name},\n{data_type},\n{nullable},\n{metadata}\n)"
+        else:
+            return f"StructField({name},\n{data_type},\n{nullable}\n)"
 
     def format_struct_type(struct_type: types.StructType) -> str:
         """

datacontract/export/sql_type_converter.py

@@ -133,8 +133,9 @@ def convert_to_dataframe(field: Field) -> None | str:
     if type.lower() in ["time"]:
         return "STRING"
     if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "DECIMAL"
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return f"DECIMAL({precision},{scale})"
     if type.lower() in ["float"]:
         return "FLOAT"
     if type.lower() in ["double"]:
@@ -182,8 +183,9 @@ def convert_to_databricks(field: Field) -> None | str:
     if type.lower() in ["time"]:
         return "STRING"
     if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "DECIMAL"
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return f"DECIMAL({precision},{scale})"
     if type.lower() in ["float"]:
         return "FLOAT"
     if type.lower() in ["double"]: