datacontract-cli 0.9.7__py3-none-any.whl → 0.9.9__py3-none-any.whl

datacontract/imports/avro_importer.py

@@ -6,19 +6,19 @@ from datacontract.model.exceptions import DataContractException
 
 
 def import_avro(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
-
     if data_contract_specification.models is None:
         data_contract_specification.models = {}
 
     try:
-        avro_schema = avro.schema.parse(open(source, "rb").read())
+        with open(source, "r") as file:
+            avro_schema = avro.schema.parse(file.read())
     except Exception as e:
         raise DataContractException(
             type="schema",
             name="Parse avro schema",
             reason=f"Failed to parse avro schema from {source}",
             engine="datacontract",
-            original_exception=e
+            original_exception=e,
         )
 
     # type record is being used for both the table and the object types in data contract
@@ -26,30 +26,83 @@ def import_avro(data_contract_specification: DataContractSpecification, source:
     fields = import_record_fields(avro_schema.fields)
 
     data_contract_specification.models[avro_schema.name] = Model(
-        type="table",
         fields=fields,
-        description=avro_schema.doc,
     )
 
+    if avro_schema.get_prop("doc") is not None:
+        data_contract_specification.models[avro_schema.name].description = avro_schema.get_prop("doc")
+
+    if avro_schema.get_prop("namespace") is not None:
+        data_contract_specification.models[avro_schema.name].namespace = avro_schema.get_prop("namespace")
+
     return data_contract_specification
 
 
 def import_record_fields(record_fields):
-
     imported_fields = {}
     for field in record_fields:
+        imported_fields[field.name] = Field()
+        imported_fields[field.name].required = True
+        imported_fields[field.name].description = field.doc
+        for prop in field.other_props:
+            imported_fields[field.name].__setattr__(prop, field.other_props[prop])
+
         if field.type.type == "record":
-            imported_fields[field.name] = Field()
             imported_fields[field.name].type = "object"
             imported_fields[field.name].description = field.type.doc
             imported_fields[field.name].fields = import_record_fields(field.type.fields)
-        else:
-            imported_fields[field.name] = Field()
+        elif field.type.type == "union":
+            imported_fields[field.name].required = False
+            type = import_type_of_optional_field(field)
+            imported_fields[field.name].type = type
+            if type == "record":
+                imported_fields[field.name].fields = import_record_fields(get_record_from_union_field(field).fields)
+        elif field.type.type == "array":
+            imported_fields[field.name].type = "array"
+            imported_fields[field.name].items = import_avro_array_items(field.type)
+        else:  # primitive type
             imported_fields[field.name].type = map_type_from_avro(field.type.type)
-            imported_fields[field.name].description = field.doc
+
     return imported_fields
 
 
+def import_avro_array_items(array_schema):
+    items = Field()
+    for prop in array_schema.other_props:
+        items.__setattr__(prop, array_schema.other_props[prop])
+
+    if array_schema.items.type == "record":
+        items.type = "object"
+        items.fields = import_record_fields(array_schema.items.fields)
+    elif array_schema.items.type == "array":
+        items.type = "array"
+        items.items = import_avro_array_items(array_schema.items)
+    else:  # primitive type
+        items.type = map_type_from_avro(array_schema.items.type)
+
+    return items
+
+
+def import_type_of_optional_field(field):
+    for field_type in field.type.schemas:
+        if field_type.type != "null":
+            return map_type_from_avro(field_type.type)
+    raise DataContractException(
+        type="schema",
+        result="failed",
+        name="Map avro type to data contract type",
+        reason="Could not import optional field: union type does not contain a non-null type",
+        engine="datacontract",
+    )
+
+
+def get_record_from_union_field(field):
+    for field_type in field.type.schemas:
+        if field_type.type == "record":
+            return field_type
+    return None
+
+
 def map_type_from_avro(avro_type_str: str):
     # TODO: ambiguous mapping in the export
     if avro_type_str == "null":
@@ -66,14 +119,8 @@ def map_type_from_avro(avro_type_str: str):
         return "long"
     elif avro_type_str == "boolean":
         return "boolean"
-    elif avro_type_str == "array":
-        raise DataContractException(
-            type="schema",
-            result="failed",
-            name="Map avro type to data contract type",
-            reason=f"Array type not supported",
-            engine="datacontract",
-        )
+    elif avro_type_str == "record":
+        return "record"
     else:
         raise DataContractException(
             type="schema",

datacontract/imports/sql_importer.py

@@ -5,12 +5,10 @@ from datacontract.model.data_contract_specification import \
 
 
 def import_sql(data_contract_specification: DataContractSpecification, format: str, source: str):
-
     ddl = parse_from_file(source, group_by_type=True)
     tables = ddl["tables"]
 
     for table in tables:
-
         if data_contract_specification.models is None:
             data_contract_specification.models = {}
 

datacontract/init/download_datacontract_file.py

@@ -9,9 +9,9 @@ def download_datacontract_file(file_path: str, from_url: str, overwrite_file: bo
 
     with requests.get(from_url) as response:
         response.raise_for_status()
-        with open(file_path, 'w') as f:
+        with open(file_path, "w") as f:
             f.write(response.text)
 
 
 class FileExistsException(Exception):
-    pass
+    pass

datacontract/integration/publish_datamesh_manager.py

@@ -1,19 +1,17 @@
-import logging
 import os
 
 import requests
 
-from datacontract.model.run import \
-    Run
+from datacontract.model.run import Run
 
 
 def publish_datamesh_manager(run: Run, publish_url: str):
     try:
         if publish_url is None:
-            url = f"https://api.datamesh-manager.com/api/runs"
+            url = "https://api.datamesh-manager.com/api/runs"
         else:
             url = publish_url
-        datamesh_manager_api_key = os.getenv('DATAMESH_MANAGER_API_KEY')
+        datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
 
         if run.dataContractId is None:
             raise Exception("Cannot publish run results, as data contract ID is unknown")
@@ -21,10 +19,7 @@ def publish_datamesh_manager(run: Run, publish_url: str):
         if datamesh_manager_api_key is None:
             raise Exception("Cannot publish run results, as DATAMESH_MANAGER_API_KEY is not set")
 
-        headers = {
-            'Content-Type': 'application/json',
-            'x-api-key': datamesh_manager_api_key
-        }
+        headers = {"Content-Type": "application/json", "x-api-key": datamesh_manager_api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
         response = requests.post(url, data=request_body, headers=headers)
@@ -33,7 +28,6 @@ def publish_datamesh_manager(run: Run, publish_url: str):
         if response.status_code != 200:
             run.log_error(f"Error publishing test results to Data Mesh Manager: {response.text}")
             return
-        logging.info("Published test results to %s", url)
+        run.log_info(f"Published test results to {url}")
     except Exception as e:
-        logging.error(f"Failed publishing test results. Error: {str(e)}")
-
+        run.log_error(f"Failed publishing test results. Error: {str(e)}")

datacontract/integration/publish_opentelemetry.py

@@ -1,21 +1,24 @@
 import logging
+import math
 import os
 from importlib import metadata
-from uuid import uuid4
-import math
 
-from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
-from opentelemetry.metrics import Observation
-
-from datacontract.model.run import \
-    Run
 from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import \
+    OTLPMetricExporter as OTLPgRPCMetricExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import \
+    OTLPMetricExporter
+from opentelemetry.metrics import Observation
 from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, \
+    PeriodicExportingMetricReader
 
-# Publishes metrics of a test run. 
-# Metric contains the values: 
-# 0 == test run passed, 
+from datacontract.model.run import Run
+
+
+# Publishes metrics of a test run.
+# Metric contains the values:
+# 0 == test run passed,
 # 1 == test run has warnings
 # 2 == test run failed
 # 3 == test run not possible due to an error
@@ -26,13 +29,14 @@ from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExpo
 # OTEL_SERVICE_NAME=datacontract-cli
 # OTEL_EXPORTER_OTLP_ENDPOINT=https://YOUR_ID.apm.westeurope.azure.elastic-cloud.com:443
 # OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20secret (Optional, when using SaaS Products)
-# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf (Optional, because it is the default value)
+# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf and OTEL_EXPORTER_OTLP_PROTOCOL=grpc
 #
 # Current limitations:
 # - no gRPC support
 # - currently, only ConsoleExporter and OTLP Exporter
 # - Metrics only, no logs yet (but loosely planned)
 
+
 def publish_opentelemetry(run: Run):
     try:
         if run.dataContractId is None:
@@ -48,7 +52,8 @@ def publish_opentelemetry(run: Run):
             name="datacontract.cli.test",
             callbacks=[lambda x: _to_observation_callback(run)],
             unit="result",
-            description="The overall result of the data contract test run")
+            description="The overall result of the data contract test run",
+        )
 
         telemetry.publish()
     except Exception as e:
@@ -80,10 +85,19 @@ def _to_observation(run):
 
 class Telemetry:
     def __init__(self):
-        self.exporter = ConsoleMetricExporter()
-        self.remote_exporter = OTLPMetricExporter()
+        protocol = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL")
+
+        # lower to allow grpc, GRPC and alike values.
+        if protocol and protocol.lower() == "grpc":
+            self.remote_exporter = OTLPgRPCMetricExporter()
+        else:
+            # Fallback to default OTEL http/protobuf which is used when the variable is not set.
+            # This Exporter also works for http/json.
+            self.remote_exporter = OTLPMetricExporter()
+
+        self.console_exporter = ConsoleMetricExporter()
         # using math.inf so it does not collect periodically. we do this in collect ourselves, one-time.
-        self.reader = PeriodicExportingMetricReader(self.exporter, export_interval_millis=math.inf)
+        self.reader = PeriodicExportingMetricReader(self.console_exporter, export_interval_millis=math.inf)
         self.remote_reader = PeriodicExportingMetricReader(self.remote_exporter, export_interval_millis=math.inf)
         provider = MeterProvider(metric_readers=[self.reader, self.remote_reader])
         metrics.set_meter_provider(provider)

datacontract/lint/files.py

@@ -10,8 +10,8 @@ def read_file(path):
             name=f"Reading data contract from {path}",
             reason=f"The file '{path}' does not exist.",
             engine="datacontract",
-            result="error"
+            result="error",
         )
-    with open(path, 'r') as file:
+    with open(path, "r") as file:
         file_content = file.read()
     return file_content

datacontract/lint/lint.py

@@ -1,10 +1,10 @@
-from enum import Enum
+import abc
 from dataclasses import dataclass, field
+from enum import Enum
 from typing import Sequence, Any, cast
-import abc
 
-from ..model.data_contract_specification import DataContractSpecification
 from datacontract.model.run import Check
+from ..model.data_contract_specification import DataContractSpecification
 
 """This module contains linter definitions for linting a data contract.
 
@@ -17,10 +17,11 @@ contract."""
 
 class LintSeverity(Enum):
     """The severity of a lint message. Generally, lint messages should be
-       emitted with a severity of ERROR. WARNING should be used when the linter
-       cannot determine a lint result, for example, when an unsupported model
-       type is used.
+    emitted with a severity of ERROR. WARNING should be used when the linter
+    cannot determine a lint result, for example, when an unsupported model
+    type is used.
     """
+
     ERROR = 2
     WARNING = 1
 
@@ -36,6 +37,7 @@ class LinterMessage:
        model: The model that caused the lint to fail. Is optional.
 
     """
+
     outcome: LintSeverity
     message: str
     model: Any = None
@@ -60,6 +62,7 @@ class LinterResult:
           results can be present in the list. An empty list means that
           the linter ran without producing warnings or errors.
     """
+
     results: Sequence[LinterMessage] = field(default_factory=list)
 
     @classmethod
@@ -72,34 +75,29 @@ class LinterResult:
 
     def with_warning(self, message, model=None):
         result = LinterMessage.warning(message, model)
-        return LinterResult(cast(list[LinterMessage],self.results) + [result])
+        return LinterResult(cast(list[LinterMessage], self.results) + [result])
 
     def with_error(self, message, model=None):
         result = LinterMessage.error(message, model)
         return LinterResult(cast(list[LinterMessage], self.results) + [result])
 
     def has_errors(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.ERROR,
-                       self.results))
+        return any(map(lambda result: result.outcome == LintSeverity.ERROR, self.results))
 
     def has_warnings(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.WARNING,
-                       self.results))
+        return any(map(lambda result: result.outcome == LintSeverity.WARNING, self.results))
 
     def error_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results
-                if result.outcome == LintSeverity.ERROR]
+        return [result for result in self.results if result.outcome == LintSeverity.ERROR]
 
     def warning_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results
-                if result.outcome == LintSeverity.WARNING]
+        return [result for result in self.results if result.outcome == LintSeverity.WARNING]
 
     def no_errors_or_warnings(self) -> bool:
         return len(self.results) == 0
 
-    def combine(self, other: 'LinterResult') -> 'LinterResult':
-        return LinterResult(cast(list[Any], self.results) +
-                            cast(list[Any], other.results))
+    def combine(self, other: "LinterResult") -> "LinterResult":
+        return LinterResult(cast(list[Any], self.results) + cast(list[Any], other.results))
 
 
 class Linter(abc.ABC):
@@ -124,23 +122,20 @@ class Linter(abc.ABC):
         result = self.lint_implementation(contract)
         checks = []
         if not result.error_results():
-            checks.append(Check(
-                type="lint",
-                name=f"Linter '{self.name}'",
-                result="passed",
-                engine="datacontract"
-            ))
+            checks.append(Check(type="lint", name=f"Linter '{self.name}'", result="passed", engine="datacontract"))
         else:
             # All linter messages are treated as warnings. Severity is
             # currently ignored, but could be used in filtering in the future
             # Linter messages with level WARNING are currently ignored, but might
             # be logged or printed in the future.
             for lint_error in result.error_results():
-                checks.append(Check(
-                    type="lint",
-                    name=f"Linter '{self.name}'",
-                    result="warning",
-                    engine="datacontract",
-                    reason=lint_error.message
-                ))
+                checks.append(
+                    Check(
+                        type="lint",
+                        name=f"Linter '{self.name}'",
+                        result="warning",
+                        engine="datacontract",
+                        reason=lint_error.message,
+                    )
+                )
         return checks

datacontract/lint/linters/description_linter.py

@@ -1,6 +1,5 @@
+from datacontract.model.data_contract_specification import DataContractSpecification
 from ..lint import Linter, LinterResult
-from datacontract.model.data_contract_specification import\
-    DataContractSpecification, Model
 
 
 class DescriptionLinter(Linter):
@@ -14,30 +13,22 @@ class DescriptionLinter(Linter):
     def id(self) -> str:
         return "description"
 
-    def lint_implementation(
-        self,
-        contract: DataContractSpecification
-    ) -> LinterResult:
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
         result = LinterResult()
         if not contract.info or not contract.info.description:
-            result = result.with_error(
-                f"Contract has empty description.")
-        for (model_name, model) in contract.models.items():
+            result = result.with_error("Contract has empty description.")
+        for model_name, model in contract.models.items():
             if not model.description:
-                result = result.with_error(
-                    f"Model '{model_name}' has empty description."
-                )
-            for (field_name, field) in model.fields.items():
+                result = result.with_error(f"Model '{model_name}' has empty description.")
+            for field_name, field in model.fields.items():
                 if not field.description:
                     result = result.with_error(
-                        f"Field '{field_name}' in model '{model_name}'"
-                        f" has empty description.")
-        for (definition_name, definition) in contract.definitions.items():
+                        f"Field '{field_name}' in model '{model_name}'" f" has empty description."
+                    )
+        for definition_name, definition in contract.definitions.items():
             if not definition.description:
-                result = result.with_error(
-                    f"Definition '{definition_name}' has empty description.")
-        for (index, example) in enumerate(contract.examples):
+                result = result.with_error(f"Definition '{definition_name}' has empty description.")
+        for index, example in enumerate(contract.examples):
             if not example.description:
-                result = result.with_error(
-                    f"Example {index + 1} has empty description.")
+                result = result.with_error(f"Example {index + 1} has empty description.")
         return result

datacontract/lint/linters/example_model_linter.py

@@ -1,14 +1,15 @@
 import csv
-import yaml
-import json
 import io
+import json
+
+import yaml
 
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Example
 from ..lint import Linter, LinterResult
-from datacontract.model.data_contract_specification import DataContractSpecification, Example
 
 
 class ExampleModelLinter(Linter):
-
     @property
     def name(self) -> str:
         return "Example(s) match model"
@@ -37,42 +38,39 @@ class ExampleModelLinter(Linter):
                     raise NotImplementedError(f"Unknown type {example.type}")
         else:
             # Checked in lint_implementation, shouldn't happen.
-            raise NotImplementedError("Can't lint object examples.") 
+            raise NotImplementedError("Can't lint object examples.")
 
-    def lint_implementation(
-        self,
-        contract: DataContractSpecification
-    ) -> LinterResult:
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
         """Check whether the example(s) headers match the model.
 
-          This linter checks whether the example's fields match the model
-          fields, and whether all required fields of the model are present in
-          the example.
+        This linter checks whether the example's fields match the model
+        fields, and whether all required fields of the model are present in
+        the example.
         """
         result = LinterResult()
         examples = contract.examples
         models = contract.models
         examples_with_model = []
-        for (index, example) in enumerate(examples):
+        for index, example in enumerate(examples):
             if example.model not in models:
-                result = result.with_error(
-                    f"Example {index + 1} has non-existent model '{example.model}'")
+                result = result.with_error(f"Example {index + 1} has non-existent model '{example.model}'")
             else:
-                examples_with_model.append(
-                    (index, example, models.get(example.model)))
-        for (index, example, model) in examples_with_model:
+                examples_with_model.append((index, example, models.get(example.model)))
+        for index, example, model in examples_with_model:
             if example.type == "custom":
-                result = result.with_warning(f"Example {index + 1} has type"
-                                             " \"custom\", cannot check model"
-                                             " conformance")
+                result = result.with_warning(
+                    f"Example {index + 1} has type" ' "custom", cannot check model' " conformance"
+                )
             elif not isinstance(example.data, str):
-                result = result.with_warning(f"Example {index + 1} is not a "
-                    "string example, can only lint string examples for now.")
+                result = result.with_warning(
+                    f"Example {index + 1} is not a " "string example, can only lint string examples for now."
+                )
             elif model.type == "object":
                 result = result.with_warning(
                     f"Example {index + 1} uses a "
                     f"model '{example.model}' with type 'object'. Linting is "
-                    "currently only supported for 'table' models")
+                    "currently only supported for 'table' models"
+                )
             else:
                 if example.type in ("csv", "yaml", "json"):
                     headers = self.get_example_headers(example)
@@ -80,13 +78,14 @@ class ExampleModelLinter(Linter):
                         if example_header not in model.fields:
                             result = result.with_error(
                                 f"Example {index + 1} has field '{example_header}'"
-                                f" that's not contained in model '{example.model}'")
-                    for (field_name, field_value) in model.fields.items():
+                                f" that's not contained in model '{example.model}'"
+                            )
+                    for field_name, field_value in model.fields.items():
                         if field_name not in headers and field_value.required:
                             result = result.with_error(
                                 f"Example {index + 1} is missing field '{field_name}'"
-                                f" required by model '{example.model}'")
+                                f" required by model '{example.model}'"
+                            )
                 else:
-                    result = result.with_error(f"Example {index + 1} has unknown type"                        
-                                               f"{example.type}")
+                    result = result.with_error(f"Example {index + 1} has unknown type" f"{example.type}")
         return result

datacontract/lint/linters/field_pattern_linter.py

@@ -1,13 +1,16 @@
 import re
 
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification
 from ..lint import Linter, LinterResult
-from datacontract.model.data_contract_specification import DataContractSpecification
+
 
 class FieldPatternLinter(Linter):
     """Checks that all patterns defined for fields are correct Python regex
-      syntax.
+    syntax.
 
     """
+
     @property
     def name(self):
         return "Field pattern is correct regex"
@@ -16,13 +19,10 @@ class FieldPatternLinter(Linter):
     def id(self) -> str:
         return "field-pattern"
 
-    def lint_implementation(
-        self,
-        contract: DataContractSpecification
-    ) -> LinterResult:
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
         result = LinterResult()
-        for (model_name, model) in contract.models.items():
-            for (field_name, field) in model.fields.items():
+        for model_name, model in contract.models.items():
+            for field_name, field in model.fields.items():
                 if field.pattern:
                     try:
                         re.compile(field.pattern)

datacontract/lint/linters/field_reference_linter.py

@@ -1,11 +1,13 @@
-from ..lint import Linter, LinterResult
 from datacontract.model.data_contract_specification import DataContractSpecification
+from ..lint import Linter, LinterResult
+
 
 class FieldReferenceLinter(Linter):
     """Checks that all references definitions in fields refer to existing
-      fields.
+    fields.
 
     """
+
     @property
     def name(self):
         return "Field references existing field"
@@ -14,24 +16,23 @@ class FieldReferenceLinter(Linter):
     def id(self) -> str:
         return "field-reference"
 
-    def lint_implementation(
-        self,
-        contract: DataContractSpecification
-    ) -> LinterResult:
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
         result = LinterResult()
-        for (model_name, model) in contract.models.items():
-            for (field_name, field) in model.fields.items():
+        for model_name, model in contract.models.items():
+            for field_name, field in model.fields.items():
                 if field.references:
                     (ref_model, ref_field) = field.references.split(".", maxsplit=2)
                     if ref_model not in contract.models:
                         result = result.with_error(
                             f"Field '{field_name}' in model '{model_name}'"
-                            f" references non-existing model '{ref_model}'.")
+                            f" references non-existing model '{ref_model}'."
+                        )
                     else:
                         ref_model_obj = contract.models[ref_model]
                         if ref_field not in ref_model_obj.fields:
                             result = result.with_error(
                                 f"Field '{field_name}' in model '{model_name}'"
                                 f" references non-existing field '{ref_field}'"
-                                f" in model '{ref_model}'.")
+                                f" in model '{ref_model}'."
+                            )
         return result