datacontract-cli 0.10.23__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/export/exporter.py

@@ -33,6 +33,7 @@ class ExportFormat(str, Enum):
     avro_idl = "avro-idl"
     sql = "sql"
     sql_query = "sql-query"
+    mermaid = "mermaid"
     html = "html"
     go = "go"
     bigquery = "bigquery"
@@ -44,6 +45,8 @@ class ExportFormat(str, Enum):
     markdown = "markdown"
     iceberg = "iceberg"
     custom = "custom"
+    excel = "excel"
+    dqx = "dqx"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/export/exporter_factory.py

@@ -89,6 +89,12 @@ exporter_factory.register_lazy_exporter(
     class_name="DbtExporter",
 )
 
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.mermaid,
+    module_path="datacontract.export.mermaid_exporter",
+    class_name="MermaidExporter",
+)
+
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.dbt_sources,
     module_path="datacontract.export.dbt_converter",
@@ -127,7 +133,7 @@ exporter_factory.register_lazy_exporter(
 
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.html,
-    module_path="datacontract.export.html_export",
+    module_path="datacontract.export.html_exporter",
     class_name="HtmlExporter",
 )
 
@@ -191,6 +197,12 @@ exporter_factory.register_lazy_exporter(
     class_name="MarkdownExporter",
 )
 
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dqx,
+    module_path="datacontract.export.dqx_converter",
+    class_name="DqxExporter",
+)
+
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
 )
@@ -198,3 +210,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.custom, module_path="datacontract.export.custom_converter", class_name="CustomExporter"
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.excel, module_path="datacontract.export.excel_exporter", class_name="ExcelExporter"
+)

datacontract/export/great_expectations_converter.py

@@ -14,11 +14,9 @@ from datacontract.export.exporter import (
     Exporter,
     _check_models_for_export,
 )
-from datacontract.export.pandas_type_converter import convert_to_pandas_type
-from datacontract.export.spark_converter import to_spark_data_type
-from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
+    DeprecatedQuality,
     Field,
     Quality,
 )
@@ -91,8 +89,14 @@ def to_great_expectations(
             model_key=model_key, contract_version=data_contract_spec.info.version
         )
     model_value = data_contract_spec.models.get(model_key)
-    quality_checks = get_quality_checks(data_contract_spec.quality)
+
+    # Support for Deprecated Quality
+    quality_checks = get_deprecated_quality_checks(data_contract_spec.quality)
+
+    expectations.extend(get_quality_checks(model_value.quality))
+
     expectations.extend(model_to_expectations(model_value.fields, engine, sql_server_type))
+
     expectations.extend(checks_to_expectations(quality_checks, model_key))
     model_expectation_suite = to_suite(expectations, expectation_suite_name)
 
@@ -135,6 +139,7 @@ def model_to_expectations(fields: Dict[str, Field], engine: str | None, sql_serv
     add_column_order_exp(fields, expectations)
     for field_name, field in fields.items():
         add_field_expectations(field_name, field, expectations, engine, sql_server_type)
+        expectations.extend(get_quality_checks(field.quality, field_name))
     return expectations
 
 
@@ -159,10 +164,16 @@ def add_field_expectations(
     """
     if field.type is not None:
         if engine == GreatExpectationsEngine.spark.value:
+            from datacontract.export.spark_converter import to_spark_data_type
+
             field_type = to_spark_data_type(field).__class__.__name__
         elif engine == GreatExpectationsEngine.pandas.value:
+            from datacontract.export.pandas_type_converter import convert_to_pandas_type
+
             field_type = convert_to_pandas_type(field)
         elif engine == GreatExpectationsEngine.sql.value:
+            from datacontract.export.sql_type_converter import convert_to_sql_type
+
             field_type = convert_to_sql_type(field, sql_server_type)
         else:
             field_type = field.type
@@ -173,6 +184,8 @@ def add_field_expectations(
         expectations.append(to_column_length_exp(field_name, field.minLength, field.maxLength))
     if field.minimum is not None or field.maximum is not None:
         expectations.append(to_column_min_max_exp(field_name, field.minimum, field.maximum))
+    if field.enum is not None and len(field.enum) != 0:
+        expectations.append(to_column_enum_exp(field_name, field.enum))
 
     return expectations
 
@@ -266,7 +279,24 @@ def to_column_min_max_exp(field_name, minimum, maximum) -> Dict[str, Any]:
     }
 
 
-def get_quality_checks(quality: Quality) -> Dict[str, Any]:
+def to_column_enum_exp(field_name, enum_list: List[str]) -> Dict[str, Any]:
+    """Creates a expect_column_values_to_be_in_set expectation.
+
+    Args:
+        field_name (str): The name of the field.
+        enum_list (Set[str]): enum list of value.
+
+    Returns:
+        Dict[str, Any]: Column value in set expectation.
+    """
+    return {
+        "expectation_type": "expect_column_values_to_be_in_set",
+        "kwargs": {"column": field_name, "value_set": enum_list},
+        "meta": {},
+    }
+
+
+def get_deprecated_quality_checks(quality: DeprecatedQuality) -> Dict[str, Any]:
     """Retrieves quality checks defined in a data contract.
 
     Args:
@@ -288,6 +318,26 @@ def get_quality_checks(quality: Quality) -> Dict[str, Any]:
     return quality_specification
 
 
+def get_quality_checks(qualities: List[Quality], field_name: str | None = None) -> List[Dict[str, Any]]:
+    """Retrieves quality checks defined in a data contract.
+
+    Args:
+        qualities (List[Quality]): List of quality object from the model specification.
+        field_name (str | None): field name if the quality list is attached to a specific field
+
+    Returns:
+        Dict[str, Any]: Dictionary of quality checks.
+    """
+    quality_specification = []
+    for quality in qualities:
+        if quality is not None and quality.engine is not None and quality.engine.lower() == "great-expectations":
+            ge_expectation = quality.implementation
+            if field_name is not None:
+                ge_expectation["column"] = field_name
+            quality_specification.append(ge_expectation)
+    return quality_specification
+
+
 def checks_to_expectations(quality_checks: Dict[str, Any], model_key: str) -> List[Dict[str, Any]]:
     """Converts quality checks to a list of expectations.
 

datacontract/export/{html_export.py → html_exporter.py}

@@ -6,8 +6,10 @@ import jinja_partials
 import pytz
 import yaml
 from jinja2 import Environment, PackageLoader, select_autoescape
+from open_data_contract_standard.model import OpenDataContractStandard
 
 from datacontract.export.exporter import Exporter
+from datacontract.export.mermaid_exporter import to_mermaid
 from datacontract.model.data_contract_specification import DataContractSpecification
 
 
@@ -16,7 +18,7 @@ class HtmlExporter(Exporter):
         return to_html(data_contract)
 
 
-def to_html(data_contract_spec: DataContractSpecification) -> str:
+def to_html(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str:
     # Load templates from templates folder
     package_loader = PackageLoader("datacontract", "templates")
     env = Environment(
@@ -31,28 +33,30 @@ def to_html(data_contract_spec: DataContractSpecification) -> str:
 
     # Load the required template
     # needs to be included in /MANIFEST.in
-    template = env.get_template("datacontract.html")
-
-    if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str):
-        quality_specification = data_contract_spec.quality.specification
-    elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object):
-        if data_contract_spec.quality.type == "great-expectations":
-            quality_specification = yaml.dump(
-                data_contract_spec.quality.specification, sort_keys=False, default_style="|"
-            )
-        else:
-            quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
-    else:
-        quality_specification = None
+    template_file = "datacontract.html"
+    if isinstance(data_contract_spec, OpenDataContractStandard):
+        template_file = "datacontract_odcs.html"
+
+    template = env.get_template(template_file)
 
     style_content, _, _ = package_loader.get_source(env, "style/output.css")
 
+    quality_specification = None
+    if isinstance(data_contract_spec, DataContractSpecification):
+        if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str):
+            quality_specification = data_contract_spec.quality.specification
+        elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object):
+            if data_contract_spec.quality.type == "great-expectations":
+                quality_specification = yaml.dump(
+                    data_contract_spec.quality.specification, sort_keys=False, default_style="|"
+                )
+            else:
+                quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False)
+
     datacontract_yaml = data_contract_spec.to_yaml()
 
-    tz = pytz.timezone("UTC")
-    now = datetime.datetime.now(tz)
-    formatted_date = now.strftime("%d %b %Y %H:%M:%S UTC")
-    datacontract_cli_version = get_version()
+    # Get the mermaid diagram
+    mermaid_diagram = to_mermaid(data_contract_spec)
 
     # Render the template with necessary data
     html_string = template.render(
@@ -60,13 +64,20 @@ def to_html(data_contract_spec: DataContractSpecification) -> str:
         quality_specification=quality_specification,
         style=style_content,
         datacontract_yaml=datacontract_yaml,
-        formatted_date=formatted_date,
-        datacontract_cli_version=datacontract_cli_version,
+        formatted_date=_formatted_date(),
+        datacontract_cli_version=get_version(),
+        mermaid_diagram=mermaid_diagram,
     )
 
     return html_string
 
 
+def _formatted_date() -> str:
+    tz = pytz.timezone("UTC")
+    now = datetime.datetime.now(tz)
+    return now.strftime("%d %b %Y %H:%M:%S UTC")
+
+
 def get_version() -> str:
     try:
         return version("datacontract_cli")

datacontract/export/markdown_converter.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, List
 
 from pydantic import BaseModel
 
@@ -12,6 +12,9 @@ from datacontract.model.data_contract_specification import (
     ServiceLevel,
 )
 
+TAB = " "
+ARROW = "↳"
+
 
 class MarkdownExporter(Exporter):
     """Exporter implementation for converting data contracts to Markdown."""
@@ -70,7 +73,8 @@ def obj_attributes_to_markdown(obj: BaseModel, excluded_fields: set = set(), is_
     else:
         bullet_char = "-"
         newline_char = "\n"
-    obj_model = obj.model_dump(exclude_unset=True, exclude=excluded_fields)
+    model_attributes_to_include = set(obj.__class__.model_fields.keys())
+    obj_model = obj.model_dump(exclude_unset=True, include=model_attributes_to_include, exclude=excluded_fields)
     description_value = obj_model.pop("description", None)
     attributes = [
         (f"{bullet_char} `{attr}`" if value is True else f"{bullet_char} **{attr}:** {value}")
@@ -78,7 +82,8 @@ def obj_attributes_to_markdown(obj: BaseModel, excluded_fields: set = set(), is_
         if value
     ]
     description = f"*{description_to_markdown(description_value)}*"
-    return newline_char.join([description] + attributes)
+    extra = [extra_to_markdown(obj, is_in_table_cell)] if obj.model_extra else []
+    return newline_char.join([description] + attributes + extra)
 
 
 def servers_to_markdown(servers: Dict[str, Server]) -> str:
@@ -153,8 +158,8 @@ def field_to_markdown(field_name: str, field: Field, level: int = 0) -> str:
     Returns:
         str: A Markdown table rows for the field.
     """
-    tabs = " " * level
-    arrow = "↳" if level > 0 else ""
+    tabs = TAB * level
+    arrow = ARROW if level > 0 else ""
     column_name = f"{tabs}{arrow} {field_name}"
 
     attributes = obj_attributes_to_markdown(field, {"type", "fields", "items", "keys", "values"}, True)
@@ -206,3 +211,127 @@ def service_level_to_markdown(service_level: ServiceLevel | None) -> str:
 
 def description_to_markdown(description: str | None) -> str:
     return (description or "No description.").replace("\n", "<br>")
+
+
+def array_of_dict_to_markdown(array: List[Dict[str, str]]) -> str:
+    """
+    Convert a list of dictionaries to a Markdown table.
+
+    Args:
+        array (List[Dict[str, str]]): A list of dictionaries where each dictionary represents a row in the table.
+
+    Returns:
+        str: A Markdown formatted table.
+    """
+    if not array:
+        return ""
+
+    headers = []
+
+    for item in array:
+        headers += item.keys()
+    headers = list(dict.fromkeys(headers))  # Preserve order and remove duplicates
+
+    markdown_parts = [
+        "| " + " | ".join(headers) + " |",
+        "| " + " | ".join(["---"] * len(headers)) + " |",
+    ]
+
+    for row in array:
+        element = row
+        markdown_parts.append(
+            "| "
+            + " | ".join(
+                f"{str(element.get(header, ''))}".replace("\n", "<br>").replace("\t", TAB) for header in headers
+            )
+            + " |"
+        )
+
+    return "\n".join(markdown_parts) + "\n"
+
+
+def array_to_markdown(array: List[str]) -> str:
+    """
+    Convert a list of strings to a Markdown formatted list.
+
+    Args:
+        array (List[str]): A list of strings to convert.
+
+    Returns:
+        str: A Markdown formatted list.
+    """
+    if not array:
+        return ""
+    return "\n".join(f"- {item}" for item in array) + "\n"
+
+
+def dict_to_markdown(dictionary: Dict[str, str]) -> str:
+    """
+    Convert a dictionary to a Markdown formatted list.
+
+    Args:
+        dictionary (Dict[str, str]): A dictionary where keys are item names and values are item descriptions.
+
+    Returns:
+        str: A Markdown formatted list of items.
+    """
+    if not dictionary:
+        return ""
+
+    markdown_parts = []
+    for key, value in dictionary.items():
+        if isinstance(value, dict):
+            markdown_parts.append(f"- {key}")
+            nested_markdown = dict_to_markdown(value)
+            if nested_markdown:
+                nested_lines = nested_markdown.split("\n")
+                for line in nested_lines:
+                    if line.strip():
+                        markdown_parts.append(f"  {line}")
+        else:
+            markdown_parts.append(f"- {key}: {value}")
+    return "\n".join(markdown_parts) + "\n"
+
+
+def extra_to_markdown(obj: BaseModel, is_in_table_cell: bool = False) -> str:
+    """
+    Convert the extra attributes of a data contract to Markdown format.
+    Args:
+        obj (BaseModel): The data contract object containing extra attributes.
+        is_in_table_cell (bool): Whether the extra attributes are in a table cell.
+    Returns:
+        str: A Markdown formatted string representing the extra attributes of the data contract.
+    """
+    extra = obj.model_extra
+
+    if not extra:
+        return ""
+
+    bullet_char = "•"
+    value_line_ending = "" if is_in_table_cell else "\n"
+    row_suffix = "<br>" if is_in_table_cell else ""
+
+    def render_header(key: str) -> str:
+        return f"{bullet_char} **{key}:** " if is_in_table_cell else f"\n### {key.capitalize()}\n"
+
+    parts: list[str] = []
+    for key_extra, value_extra in extra.items():
+        if not value_extra:
+            continue
+
+        parts.append(render_header(key_extra))
+
+        if isinstance(value_extra, list) and len(value_extra):
+            if isinstance(value_extra[0], dict):
+                parts.append(array_of_dict_to_markdown(value_extra))
+            elif isinstance(value_extra[0], str):
+                parts.append(array_to_markdown(value_extra))
+        elif isinstance(value_extra, dict):
+            parts.append(dict_to_markdown(value_extra))
+        else:
+            parts.append(f"{str(value_extra)}{value_line_ending}")
+
+        if row_suffix:
+            parts.append(row_suffix)
+
+    return "".join(parts)

datacontract/export/mermaid_exporter.py

@@ -0,0 +1,110 @@
+from open_data_contract_standard.model import OpenDataContractStandard
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification
+
+
+class MermaidExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_mermaid(data_contract)
+
+
+def to_mermaid(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str | None:
+    if isinstance(data_contract_spec, DataContractSpecification):
+        return dcs_to_mermaid(data_contract_spec)
+    elif isinstance(data_contract_spec, OpenDataContractStandard):
+        return odcs_to_mermaid(data_contract_spec)
+    else:
+        return None
+
+
+def dcs_to_mermaid(data_contract_spec: DataContractSpecification) -> str | None:
+    try:
+        if not data_contract_spec.models:
+            return None
+
+        mmd_entity = "erDiagram\n"
+        mmd_references = []
+
+        for model_name, model in data_contract_spec.models.items():
+            clean_model = _sanitize_name(model_name)
+            entity_block = ""
+
+            for field_name, field in model.fields.items():
+                clean_name = _sanitize_name(field_name)
+                field_type = field.type or "unknown"
+
+                is_pk = bool(field.primaryKey or (field.unique and field.required))
+                is_fk = bool(field.references)
+
+                entity_block += _field_line(clean_name, field_type, pk=is_pk, uk=bool(field.unique), fk=is_fk)
+
+                if field.references:
+                    references = field.references.replace(".", "·")
+                    parts = references.split("·")
+                    referenced_model = _sanitize_name(parts[0]) if len(parts) > 0 else ""
+                    referenced_field = _sanitize_name(parts[1]) if len(parts) > 1 else ""
+                    if referenced_model:
+                        label = referenced_field or clean_name
+                        mmd_references.append(f'"**{referenced_model}**" ||--o{{ "**{clean_model}**" : {label}')
+
+            mmd_entity += f'\t"**{clean_model}**" {{\n{entity_block}}}\n'
+
+        if mmd_references:
+            mmd_entity += "\n" + "\n".join(mmd_references)
+
+        return mmd_entity + "\n"
+
+    except Exception as e:
+        print(f"Error generating DCS mermaid diagram: {e}")
+        return None
+
+
+def odcs_to_mermaid(data_contract_spec: OpenDataContractStandard) -> str | None:
+    try:
+        if not data_contract_spec.schema_:
+            return None
+
+        mmd_entity = "erDiagram\n"
+
+        for schema in data_contract_spec.schema_:
+            schema_name = schema.name or schema.physicalName
+            entity_block = ""
+
+            if schema.properties:
+                for prop in schema.properties:
+                    clean_name = _sanitize_name(prop.name)
+                    indicators = ""
+
+                    if prop.primaryKey:
+                        indicators += "🔑"
+                    if getattr(prop, "partitioned", False):
+                        indicators += "🔀"
+                    if getattr(prop, "criticalDataElement", False):
+                        indicators += "⚠️"
+
+                    prop_type = prop.logicalType or prop.physicalType or "unknown"
+                    entity_block += f"\t{clean_name}{indicators} {prop_type}\n"
+
+            mmd_entity += f'\t"**{schema_name}**"' + "{\n" + entity_block + "}\n"
+
+        return f"{mmd_entity}\n"
+
+    except Exception as e:
+        print(f"Error generating ODCS mermaid diagram: {e}")
+        return None
+
+
+def _sanitize_name(name: str) -> str:
+    return name.replace("#", "Nb").replace(" ", "_").replace("/", "by")
+
+
+def _field_line(name: str, field_type: str, pk: bool = False, uk: bool = False, fk: bool = False) -> str:
+    indicators = ""
+    if pk:
+        indicators += "🔑"
+    if uk:
+        indicators += "🔒"
+    if fk:
+        indicators += "⌘"
+    return f"\t{name}{indicators} {field_type}\n"