datacontract-cli 0.10.16__py3-none-any.whl → 0.10.19__py3-none-any.whl

datacontract/imports/dbt_importer.py

@@ -3,7 +3,7 @@ from typing import TypedDict
 
 from dbt.artifacts.resources.v1.components import ColumnInfo
 from dbt.contracts.graph.manifest import Manifest
-from dbt.contracts.graph.nodes import GenericTestNode
+from dbt.contracts.graph.nodes import GenericTestNode, ManifestNode, ModelNode
 from dbt_common.contracts.constraints import ConstraintType
 
 from datacontract.imports.bigquery_importer import map_type_from_bigquery
@@ -51,6 +51,46 @@ def read_dbt_manifest(manifest_path: str) -> Manifest:
     return manifest
 
 
+def _get_primary_keys(manifest: Manifest, node: ManifestNode) -> list[str]:
+    node_unique_id = node.unique_id
+    if isinstance(node, ModelNode):
+        test_nodes = []
+        for node_id in manifest.child_map.get(node_unique_id, []):
+            test_node = manifest.nodes.get(node_id)
+            if not test_node or test_node.resource_type != "test":
+                continue
+            if not isinstance(test_node, GenericTestNode):
+                continue
+            if test_node.config.where is not None:
+                continue
+            test_nodes.append(test_node)
+        return node.infer_primary_key(test_nodes)
+    return []
+
+
+def _get_references(manifest: Manifest, node: ManifestNode) -> dict[str, str]:
+    node_unique_id = node.unique_id
+    references = {}
+    for node_id in manifest.child_map.get(node_unique_id, []):
+        test_node = manifest.nodes.get(node_id)
+        if not test_node or test_node.resource_type != "test":
+            continue
+        if not isinstance(test_node, GenericTestNode):
+            continue
+        if test_node.test_metadata.name != "relationships":
+            continue
+        if test_node.config.where is not None:
+            continue
+        if test_node.attached_node != node_unique_id:
+            continue
+        relationship_target_node_id = [n for n in test_node.depends_on.nodes if n != node_unique_id][0]
+        relationship_target_node = manifest.nodes.get(relationship_target_node_id)
+        references[f"{node.name}.{test_node.column_name}"] = (
+            f"""{relationship_target_node.name}.{test_node.test_metadata.kwargs["field"]}"""
+        )
+    return references
+
+
 def import_dbt_manifest(
     data_contract_specification: DataContractSpecification,
     manifest: Manifest,
@@ -65,28 +105,40 @@ def import_dbt_manifest(
     data_contract_specification.info.dbt_version = manifest.metadata.dbt_version
     adapter_type = manifest.metadata.adapter_type
     data_contract_specification.models = data_contract_specification.models or {}
-    for model_contents in manifest.nodes.values():
+    for node in manifest.nodes.values():
         # Only intressted in processing models.
-        if model_contents.resource_type not in resource_types:
+        if node.resource_type not in resource_types:
             continue
 
         # To allow args stored in dbt_models to filter relevant models.
         # If dbt_models is empty, use all models.
-        if dbt_nodes and model_contents.name not in dbt_nodes:
+        if dbt_nodes and node.name not in dbt_nodes:
             continue
 
+        model_unique_id = node.unique_id
+        primary_keys = _get_primary_keys(manifest, node)
+        references = _get_references(manifest, node)
+
+        primary_key = None
+        if len(primary_keys) == 1:
+            primary_key = primary_keys[0]
+
         dc_model = Model(
-            description=model_contents.description,
-            tags=model_contents.tags,
+            description=node.description,
+            tags=node.tags,
             fields=create_fields(
                 manifest,
-                model_unique_id=model_contents.unique_id,
-                columns=model_contents.columns,
+                model_unique_id=model_unique_id,
+                columns=node.columns,
+                primary_key_name=primary_key,
+                references=references,
                 adapter_type=adapter_type,
             ),
         )
+        if len(primary_keys) > 1:
+            dc_model.primaryKey = primary_keys
 
-        data_contract_specification.models[model_contents.name] = dc_model
+        data_contract_specification.models[node.name] = dc_model
 
     return data_contract_specification
 
@@ -98,9 +150,17 @@ def convert_data_type_by_adapter_type(data_type: str, adapter_type: str) -> str:
 
 
 def create_fields(
-    manifest: Manifest, model_unique_id: str, columns: dict[str, ColumnInfo], adapter_type: str
+    manifest: Manifest,
+    model_unique_id: str,
+    columns: dict[str, ColumnInfo],
+    primary_key_name: str,
+    references: dict[str, str],
+    adapter_type: str,
 ) -> dict[str, Field]:
-    fields = {column.name: create_field(manifest, model_unique_id, column, adapter_type) for column in columns.values()}
+    fields = {
+        column.name: create_field(manifest, model_unique_id, column, primary_key_name, references, adapter_type)
+        for column in columns.values()
+    }
     return fields
 
 
@@ -137,7 +197,14 @@ def get_column_tests(manifest: Manifest, model_name: str, column_name: str) -> l
     return column_tests
 
 
-def create_field(manifest: Manifest, model_unique_id: str, column: ColumnInfo, adapter_type: str) -> Field:
+def create_field(
+    manifest: Manifest,
+    model_unique_id: str,
+    column: ColumnInfo,
+    primary_key_name: str,
+    references: dict[str, str],
+    adapter_type: str,
+) -> Field:
     column_type = convert_data_type_by_adapter_type(column.data_type, adapter_type) if column.data_type else ""
     field = Field(
         description=column.description,
@@ -155,4 +222,19 @@ def create_field(manifest: Manifest, model_unique_id: str, column: ColumnInfo, a
     if required:
         field.required = required
 
+    unique = False
+    if any(constraint.type == ConstraintType.unique for constraint in column.constraints):
+        unique = True
+    if [test for test in all_tests if test["test_type"] == "unique"]:
+        unique = True
+    if unique:
+        field.unique = unique
+
+    if column.name == primary_key_name:
+        field.primaryKey = True
+
+    references_key = f"{manifest.nodes[model_unique_id].name}.{column.name}"
+    if references_key in references:
+        field.references = references[references_key]
+
     return field

datacontract/imports/importer.py

@@ -31,6 +31,7 @@ class ImportFormat(str, Enum):
     spark = "spark"
     iceberg = "iceberg"
     parquet = "parquet"
+    csv = "csv"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/imports/importer_factory.py

@@ -104,3 +104,8 @@ importer_factory.register_lazy_importer(
     module_path="datacontract.imports.parquet_importer",
     class_name="ParquetImporter",
 )
+importer_factory.register_lazy_importer(
+    name=ImportFormat.csv,
+    module_path="datacontract.imports.csv_importer",
+    class_name="CsvImporter",
+)

datacontract/imports/odcs_v2_importer.py

@@ -141,7 +141,7 @@ def import_fields(odcs_columns: Dict[str, Any], custom_type_mappings: Dict[str,
                 type=mapped_type,
                 title=column.get("businessName") if column.get("businessName") is not None else "",
                 required=not column.get("isNullable") if column.get("isNullable") is not None else False,
-                primary=column.get("isPrimary") if column.get("isPrimary") is not None else False,
+                primaryKey=column.get("isPrimary") if column.get("isPrimary") is not None else False,
                 unique=column.get("isUnique") if column.get("isUnique") is not None else False,
                 classification=column.get("classification") if column.get("classification") is not None else "",
                 tags=column.get("tags") if column.get("tags") is not None else [],

datacontract/imports/odcs_v3_importer.py

@@ -265,7 +265,7 @@ def import_fields(
                 type=mapped_type,
                 title=odcs_property.get("businessName"),
                 required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False,
-                primary=odcs_property.get("primaryKey")
+                primaryKey=odcs_property.get("primaryKey")
                 if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None
                 else False,
                 unique=odcs_property.get("unique"),

datacontract/imports/sql_importer.py

@@ -38,7 +38,7 @@ def import_sql(data_contract_specification: DataContractSpecification, format: s
             if primary_key in fields:
                 fields[primary_key].unique = True
                 fields[primary_key].required = True
-                fields[primary_key].primary = True
+                fields[primary_key].primaryKey = True
 
         data_contract_specification.models[table_name] = Model(
             type="table",

datacontract/init/init_template.py

@@ -0,0 +1,20 @@
+import importlib.resources as resources
+import logging
+
+import requests
+
+DEFAULT_DATA_CONTRACT_INIT_TEMPLATE = "datacontract-1.1.0.init.yaml"
+
+
+def get_init_template(location: str = None) -> str:
+    if location is None:
+        logging.info("Use default bundled template " + DEFAULT_DATA_CONTRACT_INIT_TEMPLATE)
+        schemas = resources.files("datacontract")
+        template = schemas.joinpath("schemas", DEFAULT_DATA_CONTRACT_INIT_TEMPLATE)
+        with template.open("r") as file:
+            return file.read()
+    elif location.startswith("http://") or location.startswith("https://"):
+        return requests.get(location).text
+    else:
+        with open(location, "r") as file:
+            return file.read()

datacontract/integration/datamesh_manager.py

@@ -2,11 +2,10 @@ import os
 
 import requests
 
-from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.model.run import Run
 
 
-def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
+def publish_test_results_to_datamesh_manager(run: Run, publish_url: str, ssl_verification: bool):
     try:
         if publish_url is None:
             # this url supports Data Mesh Manager and Data Contract Manager
@@ -28,7 +27,12 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
-        response = requests.post(url, data=request_body, headers=headers)
+        response = requests.post(
+            url,
+            data=request_body,
+            headers=headers,
+            verify=ssl_verification,
+        )
         # print("Status Code:", response.status_code)
         # print("Response Body:", response.text)
         if response.status_code != 200:
@@ -39,9 +43,12 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str):
         run.log_error(f"Failed publishing test results. Error: {str(e)}")
 
 
-def publish_data_contract_to_datamesh_manager(data_contract_specification: DataContractSpecification):
+def publish_data_contract_to_datamesh_manager(data_contract_dict: dict, ssl_verification: bool):
     try:
         api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+        host = "https://api.datamesh-manager.com"
+        if os.getenv("DATAMESH_MANAGER_HOST") is not None:
+            host = os.getenv("DATAMESH_MANAGER_HOST")
         if api_key is None:
             api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
         if api_key is None:
@@ -49,14 +56,13 @@ def publish_data_contract_to_datamesh_manager(data_contract_specification: DataC
                 "Cannot publish data contract, as neither DATAMESH_MANAGER_API_KEY nor DATACONTRACT_MANAGER_API_KEY is set"
             )
         headers = {"Content-Type": "application/json", "x-api-key": api_key}
-        spec = data_contract_specification
-        id = spec.id
-        url = "https://api.datamesh-manager.com/api/datacontracts/{0}".format(id)
-        request_body = spec.model_dump_json().encode("utf-8")
+        id = data_contract_dict["id"]
+        url = f"{host}/api/datacontracts/{id}"
         response = requests.put(
             url=url,
-            data=request_body,
+            json=data_contract_dict,
             headers=headers,
+            verify=ssl_verification,
         )
         if response.status_code != 200:
             print(f"Error publishing data contract to Data Mesh Manager: {response.text}")

datacontract/lint/linters/field_reference_linter.py

@@ -22,7 +22,16 @@ class FieldReferenceLinter(Linter):
         for model_name, model in contract.models.items():
             for field_name, field in model.fields.items():
                 if field.references:
-                    (ref_model, ref_field) = field.references.split(".", maxsplit=2)
+                    reference_hierarchy = field.references.split(".")
+                    if len(reference_hierarchy) != 2:
+                        result = result.with_error(
+                            f"Field '{field_name}' in model '{model_name}'"
+                            f" references must follow the model.field syntax and refer to a field in a model in this data contract."
+                        )
+                        continue
+                    ref_model = reference_hierarchy[0]
+                    ref_field = reference_hierarchy[1]
+
                     if ref_model not in contract.models:
                         result = result.with_error(
                             f"Field '{field_name}' in model '{model_name}'"

datacontract/lint/resolve.py

@@ -44,6 +44,27 @@ def resolve_data_contract(
         )
 
 
+def resolve_data_contract_dict(
+    data_contract_location: str = None,
+    data_contract_str: str = None,
+    data_contract: DataContractSpecification = None,
+) -> dict:
+    if data_contract_location is not None:
+        return _to_yaml(read_resource(data_contract_location))
+    elif data_contract_str is not None:
+        return _to_yaml(data_contract_str)
+    elif data_contract is not None:
+        return data_contract.model_dump()
+    else:
+        raise DataContractException(
+            type="lint",
+            result="failed",
+            name="Check that data contract YAML is valid",
+            reason="Data contract needs to be provided",
+            engine="datacontract",
+        )
+
+
 def resolve_data_contract_from_location(
     location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
@@ -54,20 +75,30 @@ def resolve_data_contract_from_location(
 def inline_definitions_into_data_contract(spec: DataContractSpecification):
     for model in spec.models.values():
         for field in model.fields.values():
-            # If ref_obj is not empty, we've already inlined definitions.
-            if not field.ref and not field.ref_obj:
-                continue
+            inline_definition_into_field(field, spec)
+
+
+def inline_definition_into_field(field, spec):
+    # iterate recursively over arrays
+    if field.items is not None:
+        inline_definition_into_field(field.items, spec)
 
-            definition = _resolve_definition_ref(field.ref, spec)
-            field.ref_obj = definition
+    # iterate recursively over nested fields
+    if field.fields is not None:
+        for nested_field_name, nested_field in field.fields.items():
+            inline_definition_into_field(nested_field, spec)
 
-            for field_name in field.model_fields.keys():
-                if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
-                    setattr(field, field_name, getattr(definition, field_name))
-            # extras
-            for extra_field_name, extra_field_value in definition.model_extra.items():
-                if extra_field_name not in field.model_extra.keys():
-                    setattr(field, extra_field_name, extra_field_value)
+    if not field.ref:
+        return
+
+    definition = _resolve_definition_ref(field.ref, spec)
+    for field_name in field.model_fields.keys():
+        if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
+            setattr(field, field_name, getattr(definition, field_name))
+    # extras
+    for extra_field_name, extra_field_value in definition.model_extra.items():
+        if extra_field_name not in field.model_extra.keys():
+            setattr(field, extra_field_name, extra_field_value)
 
 
 def _resolve_definition_ref(ref, spec) -> Definition:
@@ -202,9 +233,12 @@ def _resolve_data_contract_from_str(
     yaml_dict = _to_yaml(data_contract_str)
 
     if is_open_data_contract_standard(yaml_dict):
+        logging.info("Importing ODCS v3")
         # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
         data_contract_specification = DataContractSpecification(dataContractSpecification="1.1.0")
         return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str)
+    else:
+        logging.info("Importing DCS")
 
     _validate_data_contract_specification_schema(yaml_dict, schema_location)
     data_contract_specification = yaml_dict
@@ -218,7 +252,7 @@ def _resolve_data_contract_from_str(
     return spec
 
 
-def _to_yaml(data_contract_str):
+def _to_yaml(data_contract_str) -> dict:
     try:
         yaml_dict = yaml.safe_load(data_contract_str)
         return yaml_dict
@@ -236,7 +270,7 @@ def _to_yaml(data_contract_str):
 def _validate_data_contract_specification_schema(data_contract_yaml, schema_location: str = None):
     schema = fetch_schema(schema_location)
     try:
-        fastjsonschema.validate(schema, data_contract_yaml)
+        fastjsonschema.validate(schema, data_contract_yaml, use_default=False)
         logging.debug("YAML data is valid.")
     except JsonSchemaValueException as e:
         logging.warning(f"Data Contract YAML is invalid. Validation error: {e.message}")

datacontract/lint/schema.py

@@ -1,4 +1,6 @@
+import importlib.resources as resources
 import json
+import logging
 import os
 from typing import Any, Dict
 
@@ -6,6 +8,8 @@ import requests
 
 from datacontract.model.exceptions import DataContractException
 
+DEFAULT_DATA_CONTRACT_SCHEMA = "datacontract-1.1.0.schema.json"
+
 
 def fetch_schema(location: str = None) -> Dict[str, Any]:
     """
@@ -27,9 +31,12 @@ def fetch_schema(location: str = None) -> Dict[str, Any]:
 
     """
     if location is None:
-        location = "https://datacontract.com/datacontract.schema.json"
-
-    if location.startswith("http://") or location.startswith("https://"):
+        logging.info("Use default bundled schema " + DEFAULT_DATA_CONTRACT_SCHEMA)
+        schemas = resources.files("datacontract")
+        schema_file = schemas.joinpath("schemas", DEFAULT_DATA_CONTRACT_SCHEMA)
+        with schema_file.open("r") as file:
+            schema = json.load(file)
+    elif location.startswith("http://") or location.startswith("https://"):
         response = requests.get(location)
         schema = response.json()
     else:

datacontract/model/data_contract_specification.py

@@ -72,6 +72,7 @@ class Server(pyd.BaseModel):
     dataProductId: str = None
     outputPortId: str = None
     driver: str = None
+    storageAccount: str = None
     roles: List[ServerRole] = None
 
     model_config = pyd.ConfigDict(
@@ -112,6 +113,7 @@ class Definition(pyd.BaseModel):
     tags: List[str] = []
     links: Dict[str, str] = {}
     example: str = None
+    examples: List[Any] | None = None
 
     model_config = pyd.ConfigDict(
         extra="allow",
@@ -141,13 +143,15 @@ class Quality(pyd.BaseModel):
 
 class Field(pyd.BaseModel):
     ref: str = pyd.Field(default=None, alias="$ref")
-    ref_obj: Definition = pyd.Field(default=None, exclude=True)
     title: str | None = None
     type: str = None
     format: str = None
     required: bool = None
-    primary: bool = None
-    primaryKey: bool = None
+    primary: bool = pyd.Field(
+        default=None,
+        deprecated="Removed in Data Contract Specification v1.1.0. Use primaryKey instead.",
+    )
+    primaryKey: bool | None = None
     unique: bool | None = None
     references: str = None
     description: str | None = None
@@ -169,7 +173,10 @@ class Field(pyd.BaseModel):
     values: "Field" = None
     precision: int = None
     scale: int = None
-    example: str = None
+    example: str = pyd.Field(
+        default=None,
+        deprecated="Removed in Data Contract Specification v1.1.0. Use " "examples instead.",
+    )
     examples: List[Any] | None = None
     quality: List[Quality] | None = []
     config: Dict[str, Any] | None = None
@@ -186,6 +193,8 @@ class Model(pyd.BaseModel):
     title: Optional[str] = None
     fields: Dict[str, Field] = {}
     quality: List[Quality] | None = []
+    primaryKey: List[str] | None = []
+    examples: List[Any] | None = None
     config: Dict[str, Any] = None
     tags: List[str] | None = None
 

datacontract/model/run.py

@@ -12,6 +12,7 @@ class ResultEnum(str, Enum):
     warning = "warning"
     failed = "failed"
     error = "error"
+    info = "info"
     unknown = "unknown"
 
 

datacontract/schemas/datacontract-1.1.0.init.yaml

@@ -0,0 +1,91 @@
+dataContractSpecification: 1.1.0
+id: my-data-contract-id
+info:
+  title: My Data Contract
+  version: 0.0.1
+#  description:
+#  owner:
+#  contact:
+#    name:
+#    url:
+#    email:
+
+
+### servers
+
+#servers:
+#  production:
+#    type: s3
+#    location: s3://
+#    format: parquet
+#    delimiter: new_line
+
+### terms
+
+#terms:
+#  usage:
+#  limitations:
+#  billing:
+#  noticePeriod:
+
+
+### models
+
+# models:
+#   my_model:
+#     description:
+#     type:
+#     fields:
+#       my_field:
+#         type:
+#         description:
+
+
+### definitions
+
+# definitions:
+#   my_field:
+#     domain:
+#     name:
+#     title:
+#     type:
+#     description:
+#     example:
+#     pii:
+#     classification:
+
+
+### servicelevels
+
+#servicelevels:
+#  availability:
+#    description: The server is available during support hours
+#    percentage: 99.9%
+#  retention:
+#    description: Data is retained for one year because!
+#    period: P1Y
+#    unlimited: false
+#  latency:
+#    description: Data is available within 25 hours after the order was placed
+#    threshold: 25h
+#    sourceTimestampField: orders.order_timestamp
+#    processedTimestampField: orders.processed_timestamp
+#  freshness:
+#    description: The age of the youngest row in a table.
+#    threshold: 25h
+#    timestampField: orders.order_timestamp
+#  frequency:
+#    description: Data is delivered once a day
+#    type: batch # or streaming
+#    interval: daily # for batch, either or cron
+#    cron: 0 0 * * * # for batch, either or interval
+#  support:
+#    description: The data is available during typical business hours at headquarters
+#    time: 9am to 5pm in EST on business days
+#    responseTime: 1h
+#  backup:
+#    description: Data is backed up once a week, every Sunday at 0:00 UTC.
+#    interval: weekly
+#    cron: 0 0 * * 0
+#    recoveryTime: 24 hours
+#    recoveryPoint: 1 week