datacontract-cli 0.10.22__py3-none-any.whl → 0.10.24__py3-none-any.whl

datacontract/imports/importer.py

@@ -32,6 +32,7 @@ class ImportFormat(str, Enum):
     iceberg = "iceberg"
     parquet = "parquet"
     csv = "csv"
+    protobuf = "protobuf"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/imports/importer_factory.py

@@ -109,3 +109,8 @@ importer_factory.register_lazy_importer(
     module_path="datacontract.imports.csv_importer",
     class_name="CsvImporter",
 )
+importer_factory.register_lazy_importer(
+    name=ImportFormat.protobuf,
+    module_path="datacontract.imports.protobuf_importer",
+    class_name="ProtoBufImporter",
+)

datacontract/imports/odcs_v3_importer.py

@@ -1,5 +1,6 @@
 import datetime
 import logging
+import re
 from typing import Any, Dict, List
 from venv import logger
 
@@ -17,6 +18,7 @@ from datacontract.model.data_contract_specification import (
     Quality,
     Retention,
     Server,
+    ServerRole,
     ServiceLevel,
     Terms,
 )
@@ -73,8 +75,9 @@ def import_info(odcs_contract: Dict[str, Any]) -> Info:
         info.description = odcs_contract.get("description").get("purpose")
 
     # odcs.domain => datacontract.owner
-    if odcs_contract.get("domain") is not None:
-        info.owner = odcs_contract.get("domain")
+    owner = get_owner(odcs_contract.get("customProperties"))
+    if owner is not None:
+        info.owner = owner
 
     # add dataProduct as custom property
     if odcs_contract.get("dataProduct") is not None:
@@ -87,6 +90,17 @@ def import_info(odcs_contract: Dict[str, Any]) -> Info:
     return info
 
 
+def import_server_roles(roles: List[Dict]) -> List[ServerRole] | None:
+    if roles is None:
+        return None
+    result = []
+    for role in roles:
+        server_role = ServerRole()
+        server_role.name = role.get("role")
+        server_role.description = role.get("description")
+        result.append(server_role)
+
+
 def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None:
     if odcs_contract.get("servers") is None:
         return None
@@ -120,8 +134,8 @@ def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None:
         server.dataProductId = odcs_server.get("dataProductId")
         server.outputPortId = odcs_server.get("outputPortId")
         server.driver = odcs_server.get("driver")
-        server.roles = odcs_server.get("roles")
-
+        server.roles = import_server_roles(odcs_server.get("roles"))
+        server.storageAccount = re.search(r"(?:@|://)([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None
         servers[server_name] = server
     return servers
 
@@ -233,7 +247,7 @@ def import_field_config(odcs_property: Dict[str, Any], server_type=None) -> Dict
             config["redshiftType"] = physical_type
         elif server_type == "sqlserver":
             config["sqlserverType"] = physical_type
-        elif server_type == "databricksType":
+        elif server_type == "databricks":
             config["databricksType"] = physical_type
         else:
             config["physicalType"] = physical_type
@@ -264,7 +278,7 @@ def import_fields(
                 description=" ".join(description.splitlines()) if description is not None else None,
                 type=mapped_type,
                 title=odcs_property.get("businessName"),
-                required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False,
+                required=odcs_property.get("required") if odcs_property.get("required") is not None else None,
                 primaryKey=odcs_property.get("primaryKey")
                 if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None
                 else False,
@@ -272,11 +286,30 @@ def import_fields(
                 examples=odcs_property.get("examples") if odcs_property.get("examples") is not None else None,
                 classification=odcs_property.get("classification")
                 if odcs_property.get("classification") is not None
-                else "",
+                else None,
                 tags=odcs_property.get("tags") if odcs_property.get("tags") is not None else None,
                 quality=odcs_property.get("quality") if odcs_property.get("quality") is not None else [],
+                fields=import_fields(odcs_property.get("properties"), custom_type_mappings, server_type) 
+                if odcs_property.get("properties") is not None else {},
                 config=import_field_config(odcs_property, server_type),
+                format=odcs_property.get("format") if odcs_property.get("format") is not None else None,
             )
+            #mapped_type is array
+            if field.type == "array" and odcs_property.get("items") is not None :
+                #nested array object
+                if odcs_property.get("items").get("logicalType") == "object":
+                    field.items= Field(type="object", 
+                            fields=import_fields(odcs_property.get("items").get("properties"), custom_type_mappings, server_type))
+                #array of simple type
+                elif odcs_property.get("items").get("logicalType") is not None:
+                    field.items= Field(type = odcs_property.get("items").get("logicalType"))
+            
+            # enum from quality validValues as enum
+            if field.type == "string":
+                for q in field.quality:
+                    if hasattr(q,"validValues"):
+                        field.enum = q.validValues
+
             result[property_name] = field
         else:
             logger.info(
@@ -310,6 +343,15 @@ def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str
     return result
 
 
+def get_owner(odcs_custom_properties: List[Any]) -> str | None:
+    if odcs_custom_properties is not None:
+        for prop in odcs_custom_properties:
+            if prop["property"] == "owner":
+                return prop["value"]
+
+    return None
+
+
 def import_tags(odcs_contract) -> List[str] | None:
     if odcs_contract.get("tags") is None:
         return None

datacontract/imports/protobuf_importer.py

@@ -0,0 +1,266 @@
+import os
+import re
+import tempfile
+
+from google.protobuf import descriptor_pb2
+from grpc_tools import protoc
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.model.exceptions import DataContractException
+
+
+def map_type_from_protobuf(field_type: int):
+    protobuf_type_mapping = {
+        1: "double",
+        2: "float",
+        3: "long",
+        4: "long",  # uint64 mapped to long
+        5: "integer",  # int32 mapped to integer
+        6: "string",  # fixed64 mapped to string
+        7: "string",  # fixed32 mapped to string
+        8: "boolean",
+        9: "string",
+        12: "bytes",
+        13: "integer",  # uint32 mapped to integer
+        15: "integer",  # sfixed32 mapped to integer
+        16: "long",  # sfixed64 mapped to long
+        17: "integer",  # sint32 mapped to integer
+        18: "long",  # sint64 mapped to long
+    }
+    return protobuf_type_mapping.get(field_type, "string")
+
+
+def parse_imports(proto_file: str) -> list:
+    """
+    Parse import statements from a .proto file and return a list of imported file paths.
+    """
+    try:
+        with open(proto_file, "r") as f:
+            content = f.read()
+    except Exception as e:
+        raise DataContractException(
+            type="file",
+            name="Parse proto imports",
+            reason=f"Failed to read proto file: {proto_file}",
+            engine="datacontract",
+            original_exception=e,
+        )
+    imported_files = re.findall(r'import\s+"(.+?)";', content)
+    proto_dir = os.path.dirname(proto_file)
+    return [os.path.join(proto_dir, imp) for imp in imported_files]
+
+
+def compile_proto_to_binary(proto_files: list, output_file: str):
+    """
+    Compile the provided proto files into a single descriptor set using grpc_tools.protoc.
+    """
+    proto_dirs = set(os.path.dirname(proto) for proto in proto_files)
+    proto_paths = [f"--proto_path={d}" for d in proto_dirs]
+
+    args = [""] + proto_paths + [f"--descriptor_set_out={output_file}"] + proto_files
+    ret = protoc.main(args)
+    if ret != 0:
+        raise DataContractException(
+            type="schema",
+            name="Compile proto files",
+            reason=f"grpc_tools.protoc failed with exit code {ret}",
+            engine="datacontract",
+            original_exception=None,
+        )
+
+
+def extract_enum_values_from_fds(fds: descriptor_pb2.FileDescriptorSet, enum_name: str) -> dict:
+    """
+    Search the FileDescriptorSet for an enum definition with the given name
+    and return a dictionary of its values (name to number).
+    """
+    for file_descriptor in fds.file:
+        # Check top-level enums.
+        for enum in file_descriptor.enum_type:
+            if enum.name == enum_name:
+                return {value.name: value.number for value in enum.value}
+        # Check enums defined inside messages.
+        for message in file_descriptor.message_type:
+            for enum in message.enum_type:
+                if enum.name == enum_name:
+                    return {value.name: value.number for value in enum.value}
+    return {}
+
+
+def extract_message_fields_from_fds(fds: descriptor_pb2.FileDescriptorSet, message_name: str) -> dict:
+    """
+    Given a FileDescriptorSet and a message name, return a dict with its field definitions.
+    This function recurses for nested messages and handles enums.
+    """
+    for file_descriptor in fds.file:
+        for msg in file_descriptor.message_type:
+            if msg.name == message_name:
+                fields = {}
+                for field in msg.field:
+                    if field.type == 11:  # TYPE_MESSAGE
+                        nested_msg_name = field.type_name.split(".")[-1]
+                        nested_fields = extract_message_fields_from_fds(fds, nested_msg_name)
+                        if field.label == 3:  # repeated field
+                            field_info = {
+                                "description": f"List of {nested_msg_name}",
+                                "type": "array",
+                                "items": {"type": "object", "fields": nested_fields},
+                            }
+                        else:
+                            field_info = {
+                                "description": f"Nested object of {nested_msg_name}",
+                                "type": "object",
+                                "fields": nested_fields,
+                            }
+                    elif field.type == 14:  # TYPE_ENUM
+                        enum_name = field.type_name.split(".")[-1]
+                        enum_values = extract_enum_values_from_fds(fds, enum_name)
+                        field_info = {
+                            "description": f"Enum field {field.name}",
+                            "type": "string",
+                            "values": enum_values,
+                            "required": (field.label == 2),
+                        }
+                    else:
+                        field_info = {
+                            "description": f"Field {field.name}",
+                            "type": map_type_from_protobuf(field.type),
+                            "required": (field.label == 2),
+                        }
+                    fields[field.name] = field_info
+                return fields
+    return {}
+
+
+def import_protobuf(
+    data_contract_specification: DataContractSpecification, sources: list, import_args: dict = None
+) -> DataContractSpecification:
+    """
+    Gather all proto files (including those imported), compile them into one descriptor,
+    then generate models with nested fields and enums resolved.
+
+    The generated data contract uses generic defaults instead of specific hardcoded ones.
+    """
+
+    # --- Step 1: Gather all proto files (main and imported)
+    proto_files_set = set()
+    queue = list(sources)
+    while queue:
+        proto = queue.pop(0)
+        if proto not in proto_files_set:
+            proto_files_set.add(proto)
+            for imp in parse_imports(proto):
+                if os.path.exists(imp) and imp not in proto_files_set:
+                    queue.append(imp)
+    all_proto_files = list(proto_files_set)
+
+    # --- Step 2: Compile all proto files into a single descriptor set.
+    temp_descriptor = tempfile.NamedTemporaryFile(suffix=".pb", delete=False)
+    descriptor_file = temp_descriptor.name
+    temp_descriptor.close()  # Allow protoc to write to the file
+    try:
+        compile_proto_to_binary(all_proto_files, descriptor_file)
+
+        with open(descriptor_file, "rb") as f:
+            proto_data = f.read()
+        fds = descriptor_pb2.FileDescriptorSet()
+        try:
+            fds.ParseFromString(proto_data)
+        except Exception as e:
+            raise DataContractException(
+                type="schema",
+                name="Parse descriptor set",
+                reason="Failed to parse descriptor set from compiled proto files",
+                engine="datacontract",
+                original_exception=e,
+            )
+
+        # --- Step 3: Build models from the descriptor set.
+        all_models = {}
+        # Create a set of the main proto file basenames.
+        source_proto_basenames = {os.path.basename(proto) for proto in sources}
+
+        for file_descriptor in fds.file:
+            # Only process file descriptors that correspond to your main proto files.
+            if os.path.basename(file_descriptor.name) not in source_proto_basenames:
+                continue
+
+            for message in file_descriptor.message_type:
+                fields = {}
+                for field in message.field:
+                    if field.type == 11:  # TYPE_MESSAGE
+                        nested_msg_name = field.type_name.split(".")[-1]
+                        nested_fields = extract_message_fields_from_fds(fds, nested_msg_name)
+                        if field.label == 3:
+                            field_info = {
+                                "description": f"List of {nested_msg_name}",
+                                "type": "array",
+                                "items": {"type": "object", "fields": nested_fields},
+                            }
+                        else:
+                            field_info = {
+                                "description": f"Nested object of {nested_msg_name}",
+                                "type": "object",
+                                "fields": nested_fields,
+                            }
+                        fields[field.name] = field_info
+                    elif field.type == 14:  # TYPE_ENUM
+                        enum_name = field.type_name.split(".")[-1]
+                        enum_values = extract_enum_values_from_fds(fds, enum_name)
+                        field_info = {
+                            "description": f"Enum field {field.name}",
+                            "type": "string",
+                            "values": enum_values,
+                            "required": (field.label == 2),
+                        }
+                        fields[field.name] = field_info
+                    else:
+                        field_info = {
+                            "description": f"Field {field.name}",
+                            "type": map_type_from_protobuf(field.type),
+                            "required": (field.label == 2),
+                        }
+                        fields[field.name] = field_info
+
+                all_models[message.name] = {
+                    "description": f"Details of {message.name}.",
+                    "type": "table",
+                    "fields": fields,
+                }
+
+        data_contract_specification.models = all_models
+
+        return data_contract_specification
+    finally:
+        # Clean up the temporary descriptor file.
+        if os.path.exists(descriptor_file):
+            os.remove(descriptor_file)
+
+
+
+class ProtoBufImporter(Importer):
+    def __init__(self, name):
+        # 'name' is passed by the importer factory.
+        self.name = name
+
+    def import_source(
+        self,
+        data_contract_specification: DataContractSpecification,
+        source: str,
+        import_args: dict = None,
+    ) -> DataContractSpecification:
+        """
+        Import a protobuf file (and its imports) into the given DataContractSpecification.
+
+        Parameters:
+          - data_contract_specification: the initial specification to update.
+          - source: the protobuf file path.
+          - import_args: optional dictionary with additional arguments (e.g. 'output_dir').
+
+        Returns:
+          The updated DataContractSpecification.
+        """
+        # Wrap the source in a list because import_protobuf expects a list of sources.
+        return import_protobuf(data_contract_specification, [source], import_args)
+

datacontract/lint/resolve.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import warnings
 
 import fastjsonschema
 import yaml
@@ -16,6 +17,7 @@ from datacontract.model.data_contract_specification import (
 )
 from datacontract.model.exceptions import DataContractException
 from datacontract.model.odcs import is_open_data_contract_standard
+from datacontract.model.run import ResultEnum
 
 
 def resolve_data_contract(
@@ -37,7 +39,7 @@ def resolve_data_contract(
     else:
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason="Data contract needs to be provided",
             engine="datacontract",
@@ -58,7 +60,7 @@ def resolve_data_contract_dict(
     else:
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason="Data contract needs to be provided",
             engine="datacontract",
@@ -125,11 +127,24 @@ def _resolve_definition_ref(ref, spec) -> Definition:
         path = path.replace("file://", "")
         definition_str = _fetch_file(path)
         definition_dict = _to_yaml(definition_str)
+        if definition_path:
+            path_parts = [part for part in definition_path.split("/") if part != ""]
+            for path_part in path_parts:
+                definition_dict = definition_dict.get(path_part, None)
+                if not definition_dict:
+                    raise DataContractException(
+                        type="lint",
+                        result="failed",
+                        name="Check that data contract YAML is valid",
+                        reason=f"Cannot resolve definition {definition_path}, {path_part} not found",
+                        engine="datacontract",
+                    )
+        # this assumes that definitions_dict is a definitions dict, however,
+        # all we know is that it is a file!
         definition = Definition(**definition_dict)
-        if definition_path is not None:
-            return _find_by_path_in_definition(definition_path, definition)
-        else:
-            return definition
+        # if definition_path is not None:
+        #     definition = _find_by_path_in_definition(definition_path, definition)
+        return definition
     elif ref.startswith("#"):
         logging.info(f"Resolving definition local path {path}")
 
@@ -139,7 +154,7 @@ def _resolve_definition_ref(ref, spec) -> Definition:
     else:
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason=f"Cannot resolve reference {ref}",
             engine="datacontract",
@@ -152,7 +167,7 @@ def _find_by_path_in_spec(definition_path: str, spec: DataContractSpecification)
     if definition_key not in spec.definitions:
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason=f"Cannot resolve definition {definition_key}",
             engine="datacontract",
@@ -182,7 +197,7 @@ def _fetch_file(path) -> str:
     if not os.path.exists(path):
         raise DataContractException(
             type="export",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract definition is valid",
             reason=f"Cannot resolve reference {path}",
             engine="datacontract",
@@ -217,7 +232,7 @@ def _get_quality_ref_file(quality_spec: str | object) -> str | object:
         if not os.path.exists(ref):
             raise DataContractException(
                 type="export",
-                result="failed",
+                result=ResultEnum.failed,
                 name="Check that data contract quality is valid",
                 reason=f"Cannot resolve reference {ref}",
                 engine="datacontract",
@@ -246,8 +261,21 @@ def _resolve_data_contract_from_str(
 
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
-    if spec.quality and inline_quality:
-        _resolve_quality_ref(spec.quality)
+    ## Suppress DeprecationWarning when accessing spec.quality,
+    ## iif it is in fact *not* used.
+    with warnings.catch_warnings(record=True) as recorded_warnings:
+        spec_quality = spec.quality
+    for w in recorded_warnings:
+        if not issubclass(w.category, DeprecationWarning) or spec_quality is not None:
+            warnings.warn_explicit(
+                message=w.message,
+                category=w.category,
+                filename=w.filename,
+                lineno=w.lineno,
+                source=w.source,
+            )
+    if spec_quality and inline_quality:
+        _resolve_quality_ref(spec_quality)
 
     return spec
 

datacontract/model/data_contract_specification.py

@@ -320,8 +320,8 @@ class DataContractSpecification(pyd.BaseModel):
         return DataContractSpecification(**data)
 
     def to_yaml(self):
-        return yaml.dump(
-            self.model_dump(exclude_defaults=True, exclude_none=True, by_alias=True),
+        return yaml.safe_dump(
+            self.model_dump(mode="json", exclude_defaults=True, exclude_none=True, by_alias=True),
             sort_keys=False,
             allow_unicode=True,
         )

datacontract/model/run.py

@@ -89,6 +89,9 @@ class Run(BaseModel):
     def pretty(self):
         return self.model_dump_json(indent=2)
 
+    def pretty_logs(self) -> str:
+        return "\n".join(f"[{log.timestamp.isoformat()}] {log.level}: {log.message}" for log in self.logs)
+
     @staticmethod
     def create_run():
         """

datacontract/output/junit_test_results.py

@@ -0,0 +1,135 @@
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from xml.dom import minidom
+
+import yaml
+
+from datacontract.model.run import ResultEnum, Run
+
+
+def write_junit_test_results(run: Run, console, output_path: Path):
+    if not output_path:
+        console.print("No output path specified for JUnit test results. Skip writing JUnit test results.")
+        return
+
+    testsuite = ET.Element(
+        "testsuite",
+        id=str(run.runId),
+        name=run.dataContractId if run.dataContractId else "Data Contract",
+        tests=str(len(run.checks)),
+        errors=str(count_errors(run)),
+        failures=str(count_failed(run)),
+        skipped=str(count_skipped(run)),
+        timestamp=run.timestampStart.replace(tzinfo=None).isoformat(),
+        time=str((run.timestampEnd - run.timestampStart).total_seconds()),
+    )
+
+    testsuiteProperties = ET.SubElement(testsuite, "properties")
+    if run.dataContractId is not None:
+        ET.SubElement(testsuiteProperties, "property", name="dataContractId", value=run.dataContractId)
+    if run.dataContractVersion is not None:
+        ET.SubElement(testsuiteProperties, "property", name="dataContractVersion", value=run.dataContractVersion)
+    if run.dataProductId is not None:
+        ET.SubElement(testsuiteProperties, "property", name="dataProductId", value=run.dataProductId)
+    if run.outputPortId is not None:
+        ET.SubElement(testsuiteProperties, "property", name="outputPortId", value=run.outputPortId)
+    if run.server is not None:
+        ET.SubElement(testsuiteProperties, "property", name="server", value=run.server)
+
+    for check in run.checks:
+        testcase = ET.SubElement(testsuite, "testcase", classname=to_class_name(check), name=to_testcase_name(check))
+        if check.result == ResultEnum.passed:
+            pass
+        elif check.result == ResultEnum.failed:
+            failure = ET.SubElement(
+                testcase,
+                "failure",
+                message=check.reason if check.reason else "Failed",
+                type=check.category if check.category else "General",
+            )
+            failure.text = to_failure_text(check)
+        elif check.result == ResultEnum.error:
+            error = ET.SubElement(
+                testcase,
+                "error",
+                message=check.reason if check.reason else "Error",
+                type=check.category if check.category else "General",
+            )
+            error.text = to_failure_text(check)
+        elif check.result is ResultEnum.warning:
+            skipped = ET.SubElement(
+                testcase,
+                "skipped",
+                message=check.reason if check.reason else "Warning",
+                type=check.category if check.category else "General",
+            )
+            skipped.skipped = to_failure_text(check)
+        else:
+            ET.SubElement(
+                testcase,
+                "skipped",
+                message=check.reason if check.reason else "None",
+                type=check.category if check.category else "General",
+            )
+
+    if run.logs:
+        system_out = ET.SubElement(testsuite, "system-out")
+        system_out.text = logs_to_system_out(run)
+
+    xml_str: str = ET.tostring(testsuite, xml_declaration=True, encoding="utf-8")
+    xml_str_pretty = minidom.parseString(xml_str).toprettyxml(indent="  ")
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(xml_str_pretty)
+    console.print(f"JUnit test results written to {output_path}")
+
+
+def to_testcase_name(check):
+    if check.key:
+        return check.key
+    if check.name:
+        return check.name
+    else:
+        return "unknown"
+
+
+def logs_to_system_out(run):
+    result = ""
+    for log in run.logs:
+        result += f"{log.timestamp} {log.level}: {log.message}\n"
+    return result
+
+
+def to_class_name(check):
+    if check.model and check.field:
+        return f"{check.model}.{check.field}"
+    elif check.model:
+        return check.model
+    elif check.field:
+        return check.field
+    else:
+        return "general"
+
+
+def to_failure_text(check):
+    return (
+        f"Name: {check.name}\n"
+        f"Engine: {check.engine}\n"
+        f"Implementation:\n{check.implementation}\n\n"
+        f"Result: {check.result.value if check.result is not None else ''}\n"
+        f"Reason: {check.reason}\n"
+        f"Details: {check.details}\n"
+        f"Diagnostics:\n{yaml.dump(check.diagnostics, default_flow_style=False)}"
+    )
+
+
+def count_errors(run):
+    return sum(1 for check in run.checks if check.result == ResultEnum.error)
+
+
+def count_failed(run):
+    return sum(1 for check in run.checks if check.result == ResultEnum.failed)
+
+
+def count_skipped(run):
+    return sum(1 for check in run.checks if check.result is None)