datacontract-cli 0.10.16__py3-none-any.whl → 0.10.18__py3-none-any.whl

datacontract/export/markdown_converter.py

@@ -0,0 +1,208 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Definition,
+    Field,
+    Model,
+    Server,
+    ServiceLevel,
+)
+
+
+class MarkdownExporter(Exporter):
+    """Exporter implementation for converting data contracts to Markdown."""
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model: Model,
+        server: str,
+        sql_server_type: str,
+        export_args: dict,
+    ) -> str:
+        """Exports a data contract to Markdown format."""
+        return to_markdown(data_contract)
+
+
+def to_markdown(data_contract: DataContractSpecification) -> str:
+    """
+    Convert a data contract to its Markdown representation.
+
+    Args:
+        data_contract (DataContractSpecification): The data contract to convert.
+
+    Returns:
+        str: The Markdown representation of the data contract.
+    """
+    markdown_parts = [
+        f"# {data_contract.id}",
+        "## Info",
+        obj_attributes_to_markdown(data_contract.info),
+        "",
+        "## Servers",
+        servers_to_markdown(data_contract.servers),
+        "",
+        "## Terms",
+        obj_attributes_to_markdown(data_contract.terms),
+        "",
+        "## Models",
+        models_to_markdown(data_contract.models),
+        "",
+        "## Definitions",
+        definitions_to_markdown(data_contract.definitions),
+        "",
+        "## Service levels",
+        service_level_to_markdown(data_contract.servicelevels),
+    ]
+    return "\n".join(markdown_parts)
+
+
+def obj_attributes_to_markdown(obj: BaseModel, excluded_fields: set = set(), is_in_table_cell: bool = False) -> str:
+    if not obj:
+        return ""
+    if is_in_table_cell:
+        bullet_char = "•"
+        newline_char = "<br>"
+    else:
+        bullet_char = "-"
+        newline_char = "\n"
+    obj_model = obj.model_dump(exclude_unset=True, exclude=excluded_fields)
+    description_value = obj_model.pop("description", None)
+    attributes = [
+        (f"{bullet_char} `{attr}`" if value is True else f"{bullet_char} **{attr}:** {value}")
+        for attr, value in obj_model.items()
+        if value
+    ]
+    description = f"*{description_to_markdown(description_value)}*"
+    return newline_char.join([description] + attributes)
+
+
+def servers_to_markdown(servers: Dict[str, Server]) -> str:
+    if not servers:
+        return ""
+    markdown_parts = [
+        "| Name | Type | Attributes |",
+        "| ---- | ---- | ---------- |",
+    ]
+    for server_name, server in servers.items():
+        markdown_parts.append(
+            f"| {server_name} | {server.type or ''} | {obj_attributes_to_markdown(server, {'type'}, True)} |"
+        )
+    return "\n".join(markdown_parts)
+
+
+def models_to_markdown(models: Dict[str, Model]) -> str:
+    return "\n".join(model_to_markdown(model_name, model) for model_name, model in models.items())
+
+
+def model_to_markdown(model_name: str, model: Model) -> str:
+    """
+    Generate Markdown representation for a specific model.
+
+    Args:
+        model_name (str): The name of the model.
+        model (Model): The model object.
+
+    Returns:
+        str: The Markdown representation of the model.
+    """
+    parts = [
+        f"### {model_name}",
+        f"*{description_to_markdown(model.description)}*",
+        "",
+        "| Field | Type | Attributes |",
+        "| ----- | ---- | ---------- |",
+    ]
+
+    # Append generated field rows
+    parts.append(fields_to_markdown(model.fields))
+    return "\n".join(parts)
+
+
+def fields_to_markdown(
+    fields: Dict[str, Field],
+    level: int = 0,
+) -> str:
+    """
+    Generate Markdown table rows for all fields in a model.
+
+    Args:
+        fields (Dict[str, Field]): The fields to process.
+        level (int): The level of nesting for indentation.
+
+    Returns:
+        str: A Markdown table rows for the fields.
+    """
+
+    return "\n".join(field_to_markdown(field_name, field, level) for field_name, field in fields.items())
+
+
+def field_to_markdown(field_name: str, field: Field, level: int = 0) -> str:
+    """
+    Generate Markdown table rows for a single field, including nested structures.
+
+    Args:
+        field_name (str): The name of the field.
+        field (Field): The field object.
+        level (int): The level of nesting for indentation.
+
+    Returns:
+        str: A Markdown table rows for the field.
+    """
+    tabs = "&numsp;" * level
+    arrow = "&#x21b3;" if level > 0 else ""
+    column_name = f"{tabs}{arrow} {field_name}"
+
+    attributes = obj_attributes_to_markdown(field, {"type", "fields", "items", "keys", "values"}, True)
+
+    rows = [f"| {column_name} | {field.type} | {attributes} |"]
+
+    # Recursively handle nested fields, array, map
+    if field.fields:
+        rows.append(fields_to_markdown(field.fields, level + 1))
+    if field.items:
+        rows.append(field_to_markdown("items", field.items, level + 1))
+    if field.keys:
+        rows.append(field_to_markdown("keys", field.keys, level + 1))
+    if field.values:
+        rows.append(field_to_markdown("values", field.values, level + 1))
+
+    return "\n".join(rows)
+
+
+def definitions_to_markdown(definitions: Dict[str, Definition]) -> str:
+    if not definitions:
+        return ""
+    markdown_parts = [
+        "| Name | Type | Domain | Attributes |",
+        "| ---- | ---- | ------ | ---------- |",
+    ]
+    for definition_name, definition in definitions.items():
+        markdown_parts.append(
+            f"| {definition_name} | {definition.type or ''} | {definition.domain or ''} | {obj_attributes_to_markdown(definition, {'name', 'type', 'domain'}, True)} |",
+        )
+    return "\n".join(markdown_parts)
+
+
+def service_level_to_markdown(service_level: ServiceLevel | None) -> str:
+    if not service_level:
+        return ""
+    sections = {
+        "Availability": service_level.availability,
+        "Retention": service_level.retention,
+        "Latency": service_level.latency,
+        "Freshness": service_level.freshness,
+        "Frequency": service_level.frequency,
+        "Support": service_level.support,
+        "Backup": service_level.backup,
+    }
+    result = [f"### {name}\n{obj_attributes_to_markdown(attr)}\n" for name, attr in sections.items() if attr]
+    return "\n".join(result)
+
+
+def description_to_markdown(description: str | None) -> str:
+    return (description or "No description.").replace("\n", "<br>")

datacontract/export/odcs_v3_exporter.py

@@ -226,6 +226,12 @@ def to_property(field_name: str, field: Field) -> dict:
         property["examples"] = field.examples
     if field.example is not None:
         property["examples"] = [field.example]
+    if field.primaryKey is not None and field.primaryKey:
+        property["primaryKey"] = field.primaryKey
+        property["primaryKeyPosition"] = 1
+    if field.primary is not None and field.primary:
+        property["primaryKey"] = field.primary
+        property["primaryKeyPosition"] = 1
 
     property["customProperties"] = []
     if field.model_extra is not None:

datacontract/export/sodacl_converter.py

@@ -32,7 +32,7 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
     checks = []
     fields = model_value.fields
 
-    quote_field_name = server_type in ["postgres"]
+    quote_field_name = server_type in ["postgres", "sqlserver"]
 
     for field_name, field in fields.items():
         checks.append(check_field_is_present(field_name))

datacontract/export/sql_converter.py

@@ -113,7 +113,7 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
         result += f"  {field_name} {type}"
         if field.required:
             result += " not null"
-        if field.primary:
+        if field.primaryKey or field.primary:
             result += " primary key"
         if server_type == "databricks" and field.description is not None:
             result += f' COMMENT "{_escape(field.description)}"'

datacontract/export/sql_type_converter.py

@@ -311,6 +311,27 @@ def convert_type_to_sqlserver(field: Field) -> None | str:
 
 def convert_type_to_bigquery(field: Field) -> None | str:
     """Convert from supported datacontract types to equivalent bigquery types"""
+
+    # BigQuery exporter cannot be used for complex types, as the exporter has different syntax than SodaCL
+
+    field_type = field.type
+    if not field_type:
+        return None
+
+    if field.config and "bigqueryType" in field.config:
+        return field.config["bigqueryType"]
+
+    if field_type.lower() in ["array"]:
+        item_type = convert_type_to_bigquery(field.items)
+        return f"ARRAY<{item_type}>"
+
+    if field_type.lower() in ["object", "record", "struct"]:
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_type_to_bigquery(nested_field)
+            nested_fields.append(f"{nested_field_name} {nested_field_type}")
+        return f"STRUCT<{', '.join(nested_fields)}>"
+
     return map_type_to_bigquery(field)
 
 

datacontract/export/sqlalchemy_converter.py

@@ -114,7 +114,9 @@ def constant_field_value(field_name: str, field: spec.Field) -> tuple[ast.Call,
     if new_type is None:
         raise RuntimeError(f"Unsupported field type {field.type}.")
 
-    return Column(new_type, nullable=not field.required, comment=field.description, primary_key=field.primary), None
+    return Column(
+        new_type, nullable=not field.required, comment=field.description, primary_key=field.primaryKey or field.primary
+    ), None
 
 
 def column_assignment(field_name: str, field: spec.Field) -> tuple[ast.Call, typing.Optional[ast.ClassDef]]:

datacontract/imports/dbml_importer.py

@@ -84,7 +84,7 @@ def import_table_fields(table, references) -> dict[str, Field]:
         imported_fields[field_name] = Field()
         imported_fields[field_name].required = field.not_null
         imported_fields[field_name].description = field.note.text
-        imported_fields[field_name].primary = field.pk
+        imported_fields[field_name].primaryKey = field.pk
         imported_fields[field_name].unique = field.unique
         # This is an assumption, that these might be valid SQL Types, since
         # DBML doesn't really enforce anything other than 'no spaces' in column types

datacontract/imports/dbt_importer.py

@@ -3,7 +3,7 @@ from typing import TypedDict
 
 from dbt.artifacts.resources.v1.components import ColumnInfo
 from dbt.contracts.graph.manifest import Manifest
-from dbt.contracts.graph.nodes import GenericTestNode
+from dbt.contracts.graph.nodes import GenericTestNode, ManifestNode, ModelNode
 from dbt_common.contracts.constraints import ConstraintType
 
 from datacontract.imports.bigquery_importer import map_type_from_bigquery
@@ -51,6 +51,46 @@ def read_dbt_manifest(manifest_path: str) -> Manifest:
     return manifest
 
 
+def _get_primary_keys(manifest: Manifest, node: ManifestNode) -> list[str]:
+    node_unique_id = node.unique_id
+    if isinstance(node, ModelNode):
+        test_nodes = []
+        for node_id in manifest.child_map.get(node_unique_id, []):
+            test_node = manifest.nodes.get(node_id)
+            if not test_node or test_node.resource_type != "test":
+                continue
+            if not isinstance(test_node, GenericTestNode):
+                continue
+            if test_node.config.where is not None:
+                continue
+            test_nodes.append(test_node)
+        return node.infer_primary_key(test_nodes)
+    return []
+
+
+def _get_references(manifest: Manifest, node: ManifestNode) -> dict[str, str]:
+    node_unique_id = node.unique_id
+    references = {}
+    for node_id in manifest.child_map.get(node_unique_id, []):
+        test_node = manifest.nodes.get(node_id)
+        if not test_node or test_node.resource_type != "test":
+            continue
+        if not isinstance(test_node, GenericTestNode):
+            continue
+        if test_node.test_metadata.name != "relationships":
+            continue
+        if test_node.config.where is not None:
+            continue
+        if test_node.attached_node != node_unique_id:
+            continue
+        relationship_target_node_id = [n for n in test_node.depends_on.nodes if n != node_unique_id][0]
+        relationship_target_node = manifest.nodes.get(relationship_target_node_id)
+        references[f"{node.name}.{test_node.column_name}"] = (
+            f"""{relationship_target_node.name}.{test_node.test_metadata.kwargs["field"]}"""
+        )
+    return references
+
+
 def import_dbt_manifest(
     data_contract_specification: DataContractSpecification,
     manifest: Manifest,
@@ -65,28 +105,40 @@ def import_dbt_manifest(
     data_contract_specification.info.dbt_version = manifest.metadata.dbt_version
     adapter_type = manifest.metadata.adapter_type
     data_contract_specification.models = data_contract_specification.models or {}
-    for model_contents in manifest.nodes.values():
+    for node in manifest.nodes.values():
         # Only intressted in processing models.
-        if model_contents.resource_type not in resource_types:
+        if node.resource_type not in resource_types:
             continue
 
         # To allow args stored in dbt_models to filter relevant models.
         # If dbt_models is empty, use all models.
-        if dbt_nodes and model_contents.name not in dbt_nodes:
+        if dbt_nodes and node.name not in dbt_nodes:
             continue
 
+        model_unique_id = node.unique_id
+        primary_keys = _get_primary_keys(manifest, node)
+        references = _get_references(manifest, node)
+
+        primary_key = None
+        if len(primary_keys) == 1:
+            primary_key = primary_keys[0]
+
         dc_model = Model(
-            description=model_contents.description,
-            tags=model_contents.tags,
+            description=node.description,
+            tags=node.tags,
             fields=create_fields(
                 manifest,
-                model_unique_id=model_contents.unique_id,
-                columns=model_contents.columns,
+                model_unique_id=model_unique_id,
+                columns=node.columns,
+                primary_key_name=primary_key,
+                references=references,
                 adapter_type=adapter_type,
             ),
         )
+        if len(primary_keys) > 1:
+            dc_model.primaryKey = primary_keys
 
-        data_contract_specification.models[model_contents.name] = dc_model
+        data_contract_specification.models[node.name] = dc_model
 
     return data_contract_specification
 
@@ -98,9 +150,17 @@ def convert_data_type_by_adapter_type(data_type: str, adapter_type: str) -> str:
 
 
 def create_fields(
-    manifest: Manifest, model_unique_id: str, columns: dict[str, ColumnInfo], adapter_type: str
+    manifest: Manifest,
+    model_unique_id: str,
+    columns: dict[str, ColumnInfo],
+    primary_key_name: str,
+    references: dict[str, str],
+    adapter_type: str,
 ) -> dict[str, Field]:
-    fields = {column.name: create_field(manifest, model_unique_id, column, adapter_type) for column in columns.values()}
+    fields = {
+        column.name: create_field(manifest, model_unique_id, column, primary_key_name, references, adapter_type)
+        for column in columns.values()
+    }
     return fields
 
 
@@ -137,7 +197,14 @@ def get_column_tests(manifest: Manifest, model_name: str, column_name: str) -> l
     return column_tests
 
 
-def create_field(manifest: Manifest, model_unique_id: str, column: ColumnInfo, adapter_type: str) -> Field:
+def create_field(
+    manifest: Manifest,
+    model_unique_id: str,
+    column: ColumnInfo,
+    primary_key_name: str,
+    references: dict[str, str],
+    adapter_type: str,
+) -> Field:
     column_type = convert_data_type_by_adapter_type(column.data_type, adapter_type) if column.data_type else ""
     field = Field(
         description=column.description,
@@ -155,4 +222,19 @@ def create_field(manifest: Manifest, model_unique_id: str, column: ColumnInfo, a
     if required:
         field.required = required
 
+    unique = False
+    if any(constraint.type == ConstraintType.unique for constraint in column.constraints):
+        unique = True
+    if [test for test in all_tests if test["test_type"] == "unique"]:
+        unique = True
+    if unique:
+        field.unique = unique
+
+    if column.name == primary_key_name:
+        field.primaryKey = True
+
+    references_key = f"{manifest.nodes[model_unique_id].name}.{column.name}"
+    if references_key in references:
+        field.references = references[references_key]
+
     return field

datacontract/imports/odcs_v2_importer.py

@@ -141,7 +141,7 @@ def import_fields(odcs_columns: Dict[str, Any], custom_type_mappings: Dict[str,
                 type=mapped_type,
                 title=column.get("businessName") if column.get("businessName") is not None else "",
                 required=not column.get("isNullable") if column.get("isNullable") is not None else False,
-                primary=column.get("isPrimary") if column.get("isPrimary") is not None else False,
+                primaryKey=column.get("isPrimary") if column.get("isPrimary") is not None else False,
                 unique=column.get("isUnique") if column.get("isUnique") is not None else False,
                 classification=column.get("classification") if column.get("classification") is not None else "",
                 tags=column.get("tags") if column.get("tags") is not None else [],

datacontract/imports/odcs_v3_importer.py

@@ -265,7 +265,7 @@ def import_fields(
                 type=mapped_type,
                 title=odcs_property.get("businessName"),
                 required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False,
-                primary=odcs_property.get("primaryKey")
+                primaryKey=odcs_property.get("primaryKey")
                 if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None
                 else False,
                 unique=odcs_property.get("unique"),

datacontract/imports/sql_importer.py

@@ -38,7 +38,7 @@ def import_sql(data_contract_specification: DataContractSpecification, format: s
             if primary_key in fields:
                 fields[primary_key].unique = True
                 fields[primary_key].required = True
-                fields[primary_key].primary = True
+                fields[primary_key].primaryKey = True
 
         data_contract_specification.models[table_name] = Model(
             type="table",

datacontract/integration/datamesh_manager.py

@@ -28,7 +28,12 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
-        response = requests.post(url, data=request_body, headers=headers)
+        response = requests.post(
+            url,
+            data=request_body,
+            headers=headers,
+            verify=False,
+        )
         # print("Status Code:", response.status_code)
         # print("Response Body:", response.text)
         if response.status_code != 200:
@@ -39,9 +44,14 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
         run.log_error(f"Failed publishing test results. Error: {str(e)}")
 
 
-def publish_data_contract_to_datamesh_manager(data_contract_specification: DataContractSpecification):
+def publish_data_contract_to_datamesh_manager(
+    data_contract_specification: DataContractSpecification, ssl_verification: bool
+):
     try:
         api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+        host = "https://api.datamesh-manager.com"
+        if os.getenv("DATAMESH_MANAGER_HOST") is not None:
+            host = os.getenv("DATAMESH_MANAGER_HOST")
         if api_key is None:
             api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
         if api_key is None:
@@ -51,12 +61,13 @@ def publish_data_contract_to_datamesh_manager(data_contract_specification: DataC
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
         spec = data_contract_specification
         id = spec.id
-        url = "https://api.datamesh-manager.com/api/datacontracts/{0}".format(id)
+        url = f"{host}/api/datacontracts/{id}"
         request_body = spec.model_dump_json().encode("utf-8")
         response = requests.put(
             url=url,
             data=request_body,
             headers=headers,
+            verify=ssl_verification,
         )
         if response.status_code != 200:
             print(f"Error publishing data contract to Data Mesh Manager: {response.text}")

datacontract/lint/resolve.py

@@ -54,20 +54,30 @@ def resolve_data_contract_from_location(
 def inline_definitions_into_data_contract(spec: DataContractSpecification):
     for model in spec.models.values():
         for field in model.fields.values():
-            # If ref_obj is not empty, we've already inlined definitions.
-            if not field.ref and not field.ref_obj:
-                continue
+            inline_definition_into_field(field, spec)
 
-            definition = _resolve_definition_ref(field.ref, spec)
-            field.ref_obj = definition
 
-            for field_name in field.model_fields.keys():
-                if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
-                    setattr(field, field_name, getattr(definition, field_name))
-            # extras
-            for extra_field_name, extra_field_value in definition.model_extra.items():
-                if extra_field_name not in field.model_extra.keys():
-                    setattr(field, extra_field_name, extra_field_value)
+def inline_definition_into_field(field, spec):
+    # iterate recursively over arrays
+    if field.items is not None:
+        inline_definition_into_field(field.items, spec)
+
+    # iterate recursively over nested fields
+    if field.fields is not None:
+        for nested_field_name, nested_field in field.fields.items():
+            inline_definition_into_field(nested_field, spec)
+
+    if not field.ref:
+        return
+
+    definition = _resolve_definition_ref(field.ref, spec)
+    for field_name in field.model_fields.keys():
+        if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
+            setattr(field, field_name, getattr(definition, field_name))
+    # extras
+    for extra_field_name, extra_field_value in definition.model_extra.items():
+        if extra_field_name not in field.model_extra.keys():
+            setattr(field, extra_field_name, extra_field_value)
 
 
 def _resolve_definition_ref(ref, spec) -> Definition:
@@ -202,9 +212,12 @@ def _resolve_data_contract_from_str(
     yaml_dict = _to_yaml(data_contract_str)
 
     if is_open_data_contract_standard(yaml_dict):
+        logging.info("Importing ODCS v3")
         # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
         data_contract_specification = DataContractSpecification(dataContractSpecification="1.1.0")
         return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str)
+    else:
+        logging.info("Importing DCS")
 
     _validate_data_contract_specification_schema(yaml_dict, schema_location)
     data_contract_specification = yaml_dict
@@ -236,7 +249,7 @@ def _to_yaml(data_contract_str):
 def _validate_data_contract_specification_schema(data_contract_yaml, schema_location: str = None):
     schema = fetch_schema(schema_location)
     try:
-        fastjsonschema.validate(schema, data_contract_yaml)
+        fastjsonschema.validate(schema, data_contract_yaml, use_default=False)
         logging.debug("YAML data is valid.")
     except JsonSchemaValueException as e:
         logging.warning(f"Data Contract YAML is invalid. Validation error: {e.message}")

datacontract/model/data_contract_specification.py

@@ -141,13 +141,15 @@ class Quality(pyd.BaseModel):
 
 class Field(pyd.BaseModel):
     ref: str = pyd.Field(default=None, alias="$ref")
-    ref_obj: Definition = pyd.Field(default=None, exclude=True)
     title: str | None = None
     type: str = None
     format: str = None
     required: bool = None
-    primary: bool = None
-    primaryKey: bool = None
+    primary: bool = pyd.Field(
+        default=None,
+        deprecated="Removed in Data Contract Specification v1.1.0. Use primaryKey instead.",
+    )
+    primaryKey: bool | None = None
     unique: bool | None = None
     references: str = None
     description: str | None = None
@@ -169,7 +171,10 @@ class Field(pyd.BaseModel):
     values: "Field" = None
     precision: int = None
     scale: int = None
-    example: str = None
+    example: str = pyd.Field(
+        default=None,
+        deprecated="Removed in Data Contract Specification v1.1.0. Use " "examples instead.",
+    )
     examples: List[Any] | None = None
     quality: List[Quality] | None = []
     config: Dict[str, Any] | None = None
@@ -186,6 +191,8 @@ class Model(pyd.BaseModel):
     title: Optional[str] = None
     fields: Dict[str, Field] = {}
     quality: List[Quality] | None = []
+    primaryKey: List[str] | None = []
+    examples: List[Any] | None = None
     config: Dict[str, Any] = None
     tags: List[str] | None = None
 

datacontract/model/run.py

@@ -12,6 +12,7 @@ class ResultEnum(str, Enum):
     warning = "warning"
     failed = "failed"
     error = "error"
+    info = "info"
     unknown = "unknown"
 
 

datacontract/templates/partials/model_field.html

@@ -40,7 +40,7 @@
     {% endif %}
 
     <div>
-      {% if field.primary %}
+      {% if field.primaryKey or field.primary %}
       <span class="inline-flex items-center rounded-md bg-gray-50 px-1 py-1 text-xs font-medium text-gray-600 ring-1 ring-inset ring-gray-500/10 mr-1 mt-1">primary</span>
       {% endif %}
       {% if field.required %}