datacontract-cli 0.10.15__py3-none-any.whl → 0.10.18__py3-none-any.whl

datacontract/export/sql_type_converter.py

@@ -311,6 +311,27 @@ def convert_type_to_sqlserver(field: Field) -> None | str:
 
 def convert_type_to_bigquery(field: Field) -> None | str:
     """Convert from supported datacontract types to equivalent bigquery types"""
+
+    # BigQuery exporter cannot be used for complex types, as the exporter has different syntax than SodaCL
+
+    field_type = field.type
+    if not field_type:
+        return None
+
+    if field.config and "bigqueryType" in field.config:
+        return field.config["bigqueryType"]
+
+    if field_type.lower() in ["array"]:
+        item_type = convert_type_to_bigquery(field.items)
+        return f"ARRAY<{item_type}>"
+
+    if field_type.lower() in ["object", "record", "struct"]:
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_type_to_bigquery(nested_field)
+            nested_fields.append(f"{nested_field_name} {nested_field_type}")
+        return f"STRUCT<{', '.join(nested_fields)}>"
+
     return map_type_to_bigquery(field)
 
 

datacontract/export/sqlalchemy_converter.py

@@ -114,7 +114,9 @@ def constant_field_value(field_name: str, field: spec.Field) -> tuple[ast.Call,
     if new_type is None:
         raise RuntimeError(f"Unsupported field type {field.type}.")
 
-    return Column(new_type, nullable=not field.required, comment=field.description, primary_key=field.primary), None
+    return Column(
+        new_type, nullable=not field.required, comment=field.description, primary_key=field.primaryKey or field.primary
+    ), None
 
 
 def column_assignment(field_name: str, field: spec.Field) -> tuple[ast.Call, typing.Optional[ast.ClassDef]]:

datacontract/imports/dbml_importer.py

@@ -84,7 +84,7 @@ def import_table_fields(table, references) -> dict[str, Field]:
         imported_fields[field_name] = Field()
         imported_fields[field_name].required = field.not_null
         imported_fields[field_name].description = field.note.text
-        imported_fields[field_name].primary = field.pk
+        imported_fields[field_name].primaryKey = field.pk
         imported_fields[field_name].unique = field.unique
         # This is an assumption, that these might be valid SQL Types, since
         # DBML doesn't really enforce anything other than 'no spaces' in column types

datacontract/imports/dbt_importer.py

@@ -3,7 +3,10 @@ from typing import TypedDict
 
 from dbt.artifacts.resources.v1.components import ColumnInfo
 from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.graph.nodes import GenericTestNode, ManifestNode, ModelNode
+from dbt_common.contracts.constraints import ConstraintType
 
+from datacontract.imports.bigquery_importer import map_type_from_bigquery
 from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
 
@@ -34,7 +37,7 @@ class DbtManifestImporter(Importer):
         return import_dbt_manifest(
             data_contract_specification=data_contract_specification,
             manifest=manifest,
-            dbt_nodes=import_args.get("dbt_nodes", []),
+            dbt_nodes=import_args.get("dbt_model", []),
             resource_types=import_args.get("resource_types", ["model"]),
         )
 
@@ -43,7 +46,49 @@ def read_dbt_manifest(manifest_path: str) -> Manifest:
     """Read a manifest from file."""
     with open(file=manifest_path, mode="r", encoding="utf-8") as f:
         manifest_dict: dict = json.load(f)
-    return Manifest.from_dict(manifest_dict)
+    manifest = Manifest.from_dict(manifest_dict)
+    manifest.build_parent_and_child_maps()
+    return manifest
+
+
+def _get_primary_keys(manifest: Manifest, node: ManifestNode) -> list[str]:
+    node_unique_id = node.unique_id
+    if isinstance(node, ModelNode):
+        test_nodes = []
+        for node_id in manifest.child_map.get(node_unique_id, []):
+            test_node = manifest.nodes.get(node_id)
+            if not test_node or test_node.resource_type != "test":
+                continue
+            if not isinstance(test_node, GenericTestNode):
+                continue
+            if test_node.config.where is not None:
+                continue
+            test_nodes.append(test_node)
+        return node.infer_primary_key(test_nodes)
+    return []
+
+
+def _get_references(manifest: Manifest, node: ManifestNode) -> dict[str, str]:
+    node_unique_id = node.unique_id
+    references = {}
+    for node_id in manifest.child_map.get(node_unique_id, []):
+        test_node = manifest.nodes.get(node_id)
+        if not test_node or test_node.resource_type != "test":
+            continue
+        if not isinstance(test_node, GenericTestNode):
+            continue
+        if test_node.test_metadata.name != "relationships":
+            continue
+        if test_node.config.where is not None:
+            continue
+        if test_node.attached_node != node_unique_id:
+            continue
+        relationship_target_node_id = [n for n in test_node.depends_on.nodes if n != node_unique_id][0]
+        relationship_target_node = manifest.nodes.get(relationship_target_node_id)
+        references[f"{node.name}.{test_node.column_name}"] = (
+            f"""{relationship_target_node.name}.{test_node.test_metadata.kwargs["field"]}"""
+        )
+    return references
 
 
 def import_dbt_manifest(
@@ -58,37 +103,138 @@ def import_dbt_manifest(
     """
     data_contract_specification.info.title = manifest.metadata.project_name
     data_contract_specification.info.dbt_version = manifest.metadata.dbt_version
-
+    adapter_type = manifest.metadata.adapter_type
     data_contract_specification.models = data_contract_specification.models or {}
-    for model_contents in manifest.nodes.values():
+    for node in manifest.nodes.values():
         # Only intressted in processing models.
-        if model_contents.resource_type not in resource_types:
+        if node.resource_type not in resource_types:
             continue
 
         # To allow args stored in dbt_models to filter relevant models.
         # If dbt_models is empty, use all models.
-        if dbt_nodes and model_contents.name not in dbt_nodes:
+        if dbt_nodes and node.name not in dbt_nodes:
             continue
 
+        model_unique_id = node.unique_id
+        primary_keys = _get_primary_keys(manifest, node)
+        references = _get_references(manifest, node)
+
+        primary_key = None
+        if len(primary_keys) == 1:
+            primary_key = primary_keys[0]
+
         dc_model = Model(
-            description=model_contents.description,
-            tags=model_contents.tags,
-            fields=create_fields(columns=model_contents.columns),
+            description=node.description,
+            tags=node.tags,
+            fields=create_fields(
+                manifest,
+                model_unique_id=model_unique_id,
+                columns=node.columns,
+                primary_key_name=primary_key,
+                references=references,
+                adapter_type=adapter_type,
+            ),
         )
+        if len(primary_keys) > 1:
+            dc_model.primaryKey = primary_keys
 
-        data_contract_specification.models[model_contents.name] = dc_model
+        data_contract_specification.models[node.name] = dc_model
 
     return data_contract_specification
 
 
-def create_fields(columns: dict[str, ColumnInfo]) -> dict[str, Field]:
+def convert_data_type_by_adapter_type(data_type: str, adapter_type: str) -> str:
+    if adapter_type == "bigquery":
+        return map_type_from_bigquery(data_type)
+    return data_type
+
+
+def create_fields(
+    manifest: Manifest,
+    model_unique_id: str,
+    columns: dict[str, ColumnInfo],
+    primary_key_name: str,
+    references: dict[str, str],
+    adapter_type: str,
+) -> dict[str, Field]:
     fields = {
-        column.name: Field(
-            description=column.description,
-            type=column.data_type if column.data_type else "",
-            tags=column.tags,
-        )
+        column.name: create_field(manifest, model_unique_id, column, primary_key_name, references, adapter_type)
         for column in columns.values()
     }
-
     return fields
+
+
+def get_column_tests(manifest: Manifest, model_name: str, column_name: str) -> list[dict[str, str]]:
+    column_tests = []
+    model_node = manifest.nodes.get(model_name)
+    if not model_node:
+        raise ValueError(f"Model {model_name} not found in manifest.")
+
+    model_unique_id = model_node.unique_id
+    test_ids = manifest.child_map.get(model_unique_id, [])
+
+    for test_id in test_ids:
+        test_node = manifest.nodes.get(test_id)
+        if not test_node or test_node.resource_type != "test":
+            continue
+
+        if not isinstance(test_node, GenericTestNode):
+            continue
+
+        if test_node.column_name != column_name:
+            continue
+
+        if test_node.config.where is not None:
+            continue
+
+        column_tests.append(
+            {
+                "test_name": test_node.name,
+                "test_type": test_node.test_metadata.name,
+                "column": test_node.column_name,
+            }
+        )
+    return column_tests
+
+
+def create_field(
+    manifest: Manifest,
+    model_unique_id: str,
+    column: ColumnInfo,
+    primary_key_name: str,
+    references: dict[str, str],
+    adapter_type: str,
+) -> Field:
+    column_type = convert_data_type_by_adapter_type(column.data_type, adapter_type) if column.data_type else ""
+    field = Field(
+        description=column.description,
+        type=column_type,
+        tags=column.tags,
+    )
+
+    all_tests = get_column_tests(manifest, model_unique_id, column.name)
+
+    required = False
+    if any(constraint.type == ConstraintType.not_null for constraint in column.constraints):
+        required = True
+    if [test for test in all_tests if test["test_type"] == "not_null"]:
+        required = True
+    if required:
+        field.required = required
+
+    unique = False
+    if any(constraint.type == ConstraintType.unique for constraint in column.constraints):
+        unique = True
+    if [test for test in all_tests if test["test_type"] == "unique"]:
+        unique = True
+    if unique:
+        field.unique = unique
+
+    if column.name == primary_key_name:
+        field.primaryKey = True
+
+    references_key = f"{manifest.nodes[model_unique_id].name}.{column.name}"
+    if references_key in references:
+        field.references = references[references_key]
+
+    return field

datacontract/imports/iceberg_importer.py

@@ -42,8 +42,19 @@ def import_iceberg(
 
     model = Model(type="table", title=table_name)
 
+    # Iceberg identifier_fields aren't technically primary keys since Iceberg doesn't support primary keys,
+    # but they are close enough that we can probably treat them as primary keys on the conversion.
+    # ref: https://iceberg.apache.org/spec/#identifier-field-ids
+    # this code WILL NOT support finding nested primary key fields.
+    identifier_fields_ids = schema.identifier_field_ids
+
     for field in schema.fields:
-        model.fields[field.name] = _field_from_nested_field(field)
+        model_field = _field_from_nested_field(field)
+
+        if field.field_id in identifier_fields_ids:
+            model_field.primaryKey = True
+
+        model.fields[field.name] = model_field
 
     data_contract_specification.models[table_name] = model
     return data_contract_specification

datacontract/imports/odcs_v2_importer.py

@@ -141,7 +141,7 @@ def import_fields(odcs_columns: Dict[str, Any], custom_type_mappings: Dict[str,
                 type=mapped_type,
                 title=column.get("businessName") if column.get("businessName") is not None else "",
                 required=not column.get("isNullable") if column.get("isNullable") is not None else False,
-                primary=column.get("isPrimary") if column.get("isPrimary") is not None else False,
+                primaryKey=column.get("isPrimary") if column.get("isPrimary") is not None else False,
                 unique=column.get("isUnique") if column.get("isUnique") is not None else False,
                 classification=column.get("classification") if column.get("classification") is not None else "",
                 tags=column.get("tags") if column.get("tags") is not None else [],

datacontract/imports/odcs_v3_importer.py

@@ -14,6 +14,7 @@ from datacontract.model.data_contract_specification import (
     Field,
     Info,
     Model,
+    Quality,
     Retention,
     Server,
     ServiceLevel,
@@ -193,6 +194,10 @@ def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
         model.fields = import_fields(
             odcs_schema.get("properties"), custom_type_mappings, server_type=get_server_type(odcs_contract)
         )
+        if odcs_schema.get("quality") is not None:
+            # convert dict to pydantic model
+
+            model.quality = [Quality.model_validate(q) for q in odcs_schema.get("quality")]
         model.title = schema_name
         if odcs_schema.get("dataGranularityDescription") is not None:
             model.config = {"dataGranularityDescription": odcs_schema.get("dataGranularityDescription")}
@@ -260,7 +265,7 @@ def import_fields(
                 type=mapped_type,
                 title=odcs_property.get("businessName"),
                 required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False,
-                primary=odcs_property.get("primaryKey")
+                primaryKey=odcs_property.get("primaryKey")
                 if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None
                 else False,
                 unique=odcs_property.get("unique"),

datacontract/imports/sql_importer.py

@@ -38,7 +38,7 @@ def import_sql(data_contract_specification: DataContractSpecification, format: s
             if primary_key in fields:
                 fields[primary_key].unique = True
                 fields[primary_key].required = True
-                fields[primary_key].primary = True
+                fields[primary_key].primaryKey = True
 
         data_contract_specification.models[table_name] = Model(
             type="table",

datacontract/integration/datamesh_manager.py

@@ -28,7 +28,12 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
-        response = requests.post(url, data=request_body, headers=headers)
+        response = requests.post(
+            url,
+            data=request_body,
+            headers=headers,
+            verify=False,
+        )
         # print("Status Code:", response.status_code)
         # print("Response Body:", response.text)
         if response.status_code != 200:
@@ -39,9 +44,14 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
         run.log_error(f"Failed publishing test results. Error: {str(e)}")
 
 
-def publish_data_contract_to_datamesh_manager(data_contract_specification: DataContractSpecification):
+def publish_data_contract_to_datamesh_manager(
+    data_contract_specification: DataContractSpecification, ssl_verification: bool
+):
     try:
         api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+        host = "https://api.datamesh-manager.com"
+        if os.getenv("DATAMESH_MANAGER_HOST") is not None:
+            host = os.getenv("DATAMESH_MANAGER_HOST")
         if api_key is None:
             api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
         if api_key is None:
@@ -51,12 +61,13 @@ def publish_data_contract_to_datamesh_manager(data_contract_specification: DataC
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
         spec = data_contract_specification
         id = spec.id
-        url = "https://api.datamesh-manager.com/api/datacontracts/{0}".format(id)
+        url = f"{host}/api/datacontracts/{id}"
         request_body = spec.model_dump_json().encode("utf-8")
         response = requests.put(
             url=url,
             data=request_body,
             headers=headers,
+            verify=ssl_verification,
         )
         if response.status_code != 200:
             print(f"Error publishing data contract to Data Mesh Manager: {response.text}")

datacontract/lint/resolve.py

@@ -9,7 +9,11 @@ from datacontract.imports.odcs_v3_importer import import_odcs_v3_from_str
 from datacontract.lint.resources import read_resource
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
-from datacontract.model.data_contract_specification import DataContractSpecification, Definition, Quality
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Definition,
+    DeprecatedQuality,
+)
 from datacontract.model.exceptions import DataContractException
 from datacontract.model.odcs import is_open_data_contract_standard
 
@@ -50,20 +54,30 @@ def resolve_data_contract_from_location(
 def inline_definitions_into_data_contract(spec: DataContractSpecification):
     for model in spec.models.values():
         for field in model.fields.values():
-            # If ref_obj is not empty, we've already inlined definitions.
-            if not field.ref and not field.ref_obj:
-                continue
+            inline_definition_into_field(field, spec)
 
-            definition = _resolve_definition_ref(field.ref, spec)
-            field.ref_obj = definition
 
-            for field_name in field.model_fields.keys():
-                if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
-                    setattr(field, field_name, getattr(definition, field_name))
-            # extras
-            for extra_field_name, extra_field_value in definition.model_extra.items():
-                if extra_field_name not in field.model_extra.keys():
-                    setattr(field, extra_field_name, extra_field_value)
+def inline_definition_into_field(field, spec):
+    # iterate recursively over arrays
+    if field.items is not None:
+        inline_definition_into_field(field.items, spec)
+
+    # iterate recursively over nested fields
+    if field.fields is not None:
+        for nested_field_name, nested_field in field.fields.items():
+            inline_definition_into_field(nested_field, spec)
+
+    if not field.ref:
+        return
+
+    definition = _resolve_definition_ref(field.ref, spec)
+    for field_name in field.model_fields.keys():
+        if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
+            setattr(field, field_name, getattr(definition, field_name))
+    # extras
+    for extra_field_name, extra_field_value in definition.model_extra.items():
+        if extra_field_name not in field.model_extra.keys():
+            setattr(field, extra_field_name, extra_field_value)
 
 
 def _resolve_definition_ref(ref, spec) -> Definition:
@@ -156,7 +170,7 @@ def _fetch_file(path) -> str:
         return file.read()
 
 
-def _resolve_quality_ref(quality: Quality):
+def _resolve_quality_ref(quality: DeprecatedQuality):
     """
     Return the content of a ref file path
     @param quality data contract quality specification
@@ -198,9 +212,12 @@ def _resolve_data_contract_from_str(
     yaml_dict = _to_yaml(data_contract_str)
 
     if is_open_data_contract_standard(yaml_dict):
+        logging.info("Importing ODCS v3")
         # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
         data_contract_specification = DataContractSpecification(dataContractSpecification="1.1.0")
         return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str)
+    else:
+        logging.info("Importing DCS")
 
     _validate_data_contract_specification_schema(yaml_dict, schema_location)
     data_contract_specification = yaml_dict
@@ -232,7 +249,7 @@ def _to_yaml(data_contract_str):
 def _validate_data_contract_specification_schema(data_contract_yaml, schema_location: str = None):
     schema = fetch_schema(schema_location)
     try:
-        fastjsonschema.validate(schema, data_contract_yaml)
+        fastjsonschema.validate(schema, data_contract_yaml, use_default=False)
         logging.debug("YAML data is valid.")
     except JsonSchemaValueException as e:
         logging.warning(f"Data Contract YAML is invalid. Validation error: {e.message}")

datacontract/model/data_contract_specification.py

@@ -141,13 +141,15 @@ class Quality(pyd.BaseModel):
 
 class Field(pyd.BaseModel):
     ref: str = pyd.Field(default=None, alias="$ref")
-    ref_obj: Definition = pyd.Field(default=None, exclude=True)
     title: str | None = None
     type: str = None
     format: str = None
     required: bool = None
-    primary: bool = None
-    primaryKey: bool = None
+    primary: bool = pyd.Field(
+        default=None,
+        deprecated="Removed in Data Contract Specification v1.1.0. Use primaryKey instead.",
+    )
+    primaryKey: bool | None = None
     unique: bool | None = None
     references: str = None
     description: str | None = None
@@ -169,7 +171,10 @@ class Field(pyd.BaseModel):
     values: "Field" = None
     precision: int = None
     scale: int = None
-    example: str = None
+    example: str = pyd.Field(
+        default=None,
+        deprecated="Removed in Data Contract Specification v1.1.0. Use " "examples instead.",
+    )
     examples: List[Any] | None = None
     quality: List[Quality] | None = []
     config: Dict[str, Any] | None = None
@@ -186,6 +191,8 @@ class Model(pyd.BaseModel):
     title: Optional[str] = None
     fields: Dict[str, Field] = {}
     quality: List[Quality] | None = []
+    primaryKey: List[str] | None = []
+    examples: List[Any] | None = None
     config: Dict[str, Any] = None
     tags: List[str] | None = None
 
@@ -214,7 +221,8 @@ class Example(pyd.BaseModel):
     data: str | object = None
 
 
-class Quality(pyd.BaseModel):
+# Deprecated Quality class
+class DeprecatedQuality(pyd.BaseModel):
     type: str = None
     specification: str | object = None
 
@@ -287,7 +295,7 @@ class DataContractSpecification(pyd.BaseModel):
         default_factory=list,
         deprecated="Removed in Data Contract Specification " "v1.1.0. Use models.examples instead.",
     )
-    quality: Quality = pyd.Field(
+    quality: DeprecatedQuality = pyd.Field(
         default=None,
         deprecated="Removed in Data Contract Specification v1.1.0. Use " "model-level and field-level quality instead.",
     )

datacontract/model/run.py

@@ -12,6 +12,7 @@ class ResultEnum(str, Enum):
     warning = "warning"
     failed = "failed"
     error = "error"
+    info = "info"
     unknown = "unknown"
 
 

datacontract/templates/partials/model_field.html

@@ -40,7 +40,7 @@
     {% endif %}
 
     <div>
-      {% if field.primary %}
+      {% if field.primaryKey or field.primary %}
       <span class="inline-flex items-center rounded-md bg-gray-50 px-1 py-1 text-xs font-medium text-gray-600 ring-1 ring-inset ring-gray-500/10 mr-1 mt-1">primary</span>
       {% endif %}
       {% if field.required %}