datacontract-cli 0.9.7__py3-none-any.whl → 0.9.8__py3-none-any.whl

datacontract/export/sodacl_converter.py

@@ -1,14 +1,15 @@
 import yaml
 
+from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import \
     DataContractSpecification
 
 
-def to_sodacl_yaml(data_contract_spec: DataContractSpecification, check_types: bool = True) -> str:
+def to_sodacl_yaml(data_contract_spec: DataContractSpecification, server_type: str = None, check_types: bool = True) -> str:
     try:
         sodacl = {}
         for model_key, model_value in data_contract_spec.models.items():
-            k, v = to_checks(model_key, model_value, check_types)
+            k, v = to_checks(model_key, model_value, server_type, check_types)
             sodacl[k] = v
         add_quality_checks(sodacl, data_contract_spec)
         sodacl_yaml_str = yaml.dump(sodacl, default_flow_style=False, sort_keys=False)
@@ -17,17 +18,21 @@ def to_sodacl_yaml(data_contract_spec: DataContractSpecification, check_types: b
         return f"Error: {e}"
 
 
-def to_checks(model_key, model_value, check_types: bool):
+def to_checks(model_key, model_value, server_type: str, check_types: bool):
     checks = []
     fields = model_value.fields
+
+    quote_field_name = server_type in ["postgres"]
+
     for field_name, field in fields.items():
         checks.append(check_field_is_present(field_name))
         if check_types and field.type is not None:
-            checks.append(check_field_type(field_name, field.type))
+            sql_type = convert_to_sql_type(field, server_type)
+            checks.append(check_field_type(field_name, sql_type))
         if field.required:
-            checks.append(check_field_required(field_name))
+            checks.append(check_field_required(field_name, quote_field_name))
         if field.unique:
-            checks.append(check_field_unique(field_name))
+            checks.append(check_field_unique(field_name, quote_field_name))
 
     return f"checks for {model_key}", checks
 
@@ -37,10 +42,8 @@ def check_field_is_present(field_name):
         "schema": {
             "name": f"Check that field {field_name} is present",
             "fail": {
-                "when required column missing": [
-                    field_name
-                ],
-            }
+                "when required column missing": [field_name],
+            },
         }
     }
 
@@ -49,28 +52,25 @@ def check_field_type(field_name: str, type: str):
     return {
         "schema": {
             "name": f"Check that field {field_name} has type {type}",
-            "fail": {
-                "when wrong column type": {
-                    field_name: type
-                }
-            }
+            "fail": {"when wrong column type": {field_name: type}},
         }
     }
 
 
-def check_field_required(field_name):
+def check_field_required(field_name: str, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f"\"{field_name}\""
+
     return {
-        f"missing_count({field_name}) = 0": {
-            "name": f"Check that required field {field_name} has no null values"
-        }
-    }
+        f"missing_count({field_name}) = 0": {"name": f"Check that required field {field_name} has no null values"}}
 
 
-def check_field_unique(field_name):
+def check_field_unique(field_name, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f"\"{field_name}\""
     return {
-        f'duplicate_count({field_name}) = 0': {
-            "name": f"Check that unique field {field_name} has no duplicate values"
-        }
+        f"duplicate_count({field_name}) = 0": {
+            "name": f"Check that unique field {field_name} has no duplicate values"}
     }
 
 

datacontract/export/sql_converter.py

@@ -1,9 +1,10 @@
 from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model
+from datacontract.model.data_contract_specification import DataContractSpecification, Model
 
 
-def to_sql_query(data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake") -> str:
+def to_sql_query(
+    data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake"
+) -> str:
     if data_contract_spec is None:
         return ""
     if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
@@ -42,27 +43,39 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str =
     if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
         return ""
 
+    table_prefix = ""
+
     for server_name, server in iter(data_contract_spec.servers.items()):
-        if server.type == server_type:
-            break
         if server.type == "snowflake":
             server_type = "snowflake"
             break
         if server.type == "postgres":
             server_type = "postgres"
             break
+        if server.type == "databricks":
+            server_type = "databricks"
+            if server.catalog is not None and server.schema_ is not None:
+                table_prefix = server.catalog + "." + server.schema_ + "."
+            break
+        if server.type == server_type:
+            break
 
     result = ""
     result += f"-- Data Contract: {data_contract_spec.id}\n"
     result += f"-- SQL Dialect: {server_type}\n"
     for model_name, model in iter(data_contract_spec.models.items()):
-        result += _to_sql_table(model_name, model, server_type)
+        result += _to_sql_table(table_prefix + model_name, model, server_type)
 
     return result.strip()
 
 
 def _to_sql_table(model_name, model, server_type="snowflake"):
-    result = f"CREATE TABLE {model_name} (\n"
+    if server_type == "databricks":
+        # Databricks recommends to use the CREATE OR REPLACE statement for unity managed tables
+        # https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html
+        result = f"CREATE OR REPLACE TABLE {model_name} (\n"
+    else:
+        result = f"CREATE TABLE {model_name} (\n"
     fields = len(model.fields)
     current_field_index = 1
     for field_name, field in iter(model.fields.items()):
@@ -78,5 +91,3 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
         current_field_index += 1
     result += ");\n"
     return result
-
-

datacontract/export/sql_type_converter.py

@@ -6,7 +6,10 @@ def convert_to_sql_type(field: Field, server_type: str) -> str:
         return convert_to_snowflake(field)
     if server_type == "postgres":
         return convert_type_to_postgres(field)
-    return str(type)
+    if server_type == "databricks":
+        return convert_to_databricks(field)
+    return field.type
+
 
 # snowflake data types:
 # https://docs.snowflake.com/en/sql-reference/data-types.html
@@ -48,17 +51,16 @@ def convert_to_snowflake(field) -> None | str:
     return None
 
 
-
 # https://www.postgresql.org/docs/current/datatype.html
 # Using the name whenever possible
-def convert_type_to_postgres(field : Field) -> None | str:
+def convert_type_to_postgres(field: Field) -> None | str:
     type = field.type
     if type is None:
         return None
     if type.lower() in ["string", "varchar", "text"]:
         if field.format == "uuid":
             return "uuid"
-        return "text" # STRING does not exist, TEXT and VARCHAR are all the same in postrges
+        return "text"  # STRING does not exist, TEXT and VARCHAR are all the same in postrges
     if type.lower() in ["timestamp", "timestamp_tz"]:
         return "timestamptz"
     if type.lower() in ["timestamp_ntz"]:
@@ -89,3 +91,41 @@ def convert_type_to_postgres(field : Field) -> None | str:
     if type.lower() in ["array"]:
         return convert_to_sql_type(field.items, "postgres") + "[]"
     return None
+
+
+# databricks data types:
+# https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
+def convert_to_databricks(field) -> None | str:
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "STRING"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP_NTZ"
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "STRING"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "DECIMAL"
+    if type.lower() in ["float"]:
+        return "FLOAT"
+    if type.lower() in ["double"]:
+        return "DOUBLE"
+    if type.lower() in ["integer", "int"]:
+        return "INT"
+    if type.lower() in ["long", "bigint"]:
+        return "BIGINT"
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        return "STRUCT"
+    if type.lower() in ["bytes"]:
+        return "BINARY"
+    if type.lower() in ["array"]:
+        return "ARRAY"
+    return None

datacontract/export/terraform_converter.py

@@ -1,6 +1,7 @@
+import re
+
 from datacontract.model.data_contract_specification import \
     DataContractSpecification, Server
-import re
 
 
 def to_terraform(data_contract_spec: DataContractSpecification, server_id: str = None) -> str:
@@ -18,7 +19,7 @@ def to_terraform(data_contract_spec: DataContractSpecification, server_id: str =
     return result.strip()
 
 
-def server_to_terraform_resource(data_contract_spec, result, server : Server, server_name):
+def server_to_terraform_resource(data_contract_spec, result, server: Server, server_name):
     tag_data_contract = data_contract_spec.id
     tag_name = data_contract_spec.info.title
     tag_server = server_name
@@ -60,7 +61,7 @@ def extract_bucket_name(server) -> str | None:
     if server.type == "s3":
         s3_url = server.location
         # Regular expression to match the S3 bucket name
-        match = re.search(r's3://([^/]+)/', s3_url)
+        match = re.search(r"s3://([^/]+)/", s3_url)
         if match:
             # Return the first group (bucket name)
             return match.group(1)

datacontract/imports/avro_importer.py

@@ -6,19 +6,19 @@ from datacontract.model.exceptions import DataContractException
 
 
 def import_avro(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
-
     if data_contract_specification.models is None:
         data_contract_specification.models = {}
 
     try:
-        avro_schema = avro.schema.parse(open(source, "rb").read())
+        with open(source, "r") as file:
+            avro_schema = avro.schema.parse(file.read())
     except Exception as e:
         raise DataContractException(
             type="schema",
             name="Parse avro schema",
             reason=f"Failed to parse avro schema from {source}",
             engine="datacontract",
-            original_exception=e
+            original_exception=e,
         )
 
     # type record is being used for both the table and the object types in data contract
@@ -28,28 +28,50 @@ def import_avro(data_contract_specification: DataContractSpecification, source:
     data_contract_specification.models[avro_schema.name] = Model(
         type="table",
         fields=fields,
-        description=avro_schema.doc,
     )
 
+    if avro_schema.get_prop("doc") is not None:
+        data_contract_specification.models[avro_schema.name].description = avro_schema.get_prop("doc")
+
+    if avro_schema.get_prop("namespace") is not None:
+        data_contract_specification.models[avro_schema.name].namespace = avro_schema.get_prop("namespace")
+
     return data_contract_specification
 
 
 def import_record_fields(record_fields):
-
     imported_fields = {}
     for field in record_fields:
+
+        imported_fields[field.name] = Field()
+        imported_fields[field.name].required = True
+        imported_fields[field.name].description = field.doc
+
         if field.type.type == "record":
-            imported_fields[field.name] = Field()
             imported_fields[field.name].type = "object"
             imported_fields[field.name].description = field.type.doc
             imported_fields[field.name].fields = import_record_fields(field.type.fields)
-        else:
-            imported_fields[field.name] = Field()
+        elif field.type.type == "union":
+            imported_fields[field.name].required = False
+            imported_fields[field.name].type = import_type_of_optional_field(field)
+        else: # primitive type
             imported_fields[field.name].type = map_type_from_avro(field.type.type)
-            imported_fields[field.name].description = field.doc
     return imported_fields
 
 
+def import_type_of_optional_field(field):
+    for field_type in field.type.schemas:
+        if field_type.type != "null":
+            return map_type_from_avro(field_type.type)
+    raise DataContractException(
+        type="schema",
+        result="failed",
+        name="Map avro type to data contract type",
+        reason="Could not import optional field: union type does not contain a non-null type",
+        engine="datacontract",
+    )
+
+
 def map_type_from_avro(avro_type_str: str):
     # TODO: ambiguous mapping in the export
     if avro_type_str == "null":
@@ -71,7 +93,7 @@ def map_type_from_avro(avro_type_str: str):
             type="schema",
             result="failed",
             name="Map avro type to data contract type",
-            reason=f"Array type not supported",
+            reason="Array type not supported",
             engine="datacontract",
         )
     else:

datacontract/imports/sql_importer.py

@@ -5,12 +5,10 @@ from datacontract.model.data_contract_specification import \
 
 
 def import_sql(data_contract_specification: DataContractSpecification, format: str, source: str):
-
     ddl = parse_from_file(source, group_by_type=True)
     tables = ddl["tables"]
 
     for table in tables:
-
         if data_contract_specification.models is None:
             data_contract_specification.models = {}
 

datacontract/init/download_datacontract_file.py

@@ -9,9 +9,9 @@ def download_datacontract_file(file_path: str, from_url: str, overwrite_file: bo
 
     with requests.get(from_url) as response:
         response.raise_for_status()
-        with open(file_path, 'w') as f:
+        with open(file_path, "w") as f:
             f.write(response.text)
 
 
 class FileExistsException(Exception):
-    pass
+    pass

datacontract/integration/publish_datamesh_manager.py

@@ -3,17 +3,16 @@ import os
 
 import requests
 
-from datacontract.model.run import \
-    Run
+from datacontract.model.run import Run
 
 
 def publish_datamesh_manager(run: Run, publish_url: str):
     try:
         if publish_url is None:
-            url = f"https://api.datamesh-manager.com/api/runs"
+            url = "https://api.datamesh-manager.com/api/runs"
         else:
             url = publish_url
-        datamesh_manager_api_key = os.getenv('DATAMESH_MANAGER_API_KEY')
+        datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
 
         if run.dataContractId is None:
             raise Exception("Cannot publish run results, as data contract ID is unknown")
@@ -21,10 +20,7 @@ def publish_datamesh_manager(run: Run, publish_url: str):
         if datamesh_manager_api_key is None:
             raise Exception("Cannot publish run results, as DATAMESH_MANAGER_API_KEY is not set")
 
-        headers = {
-            'Content-Type': 'application/json',
-            'x-api-key': datamesh_manager_api_key
-        }
+        headers = {"Content-Type": "application/json", "x-api-key": datamesh_manager_api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
         response = requests.post(url, data=request_body, headers=headers)
@@ -36,4 +32,3 @@ def publish_datamesh_manager(run: Run, publish_url: str):
         logging.info("Published test results to %s", url)
     except Exception as e:
         logging.error(f"Failed publishing test results. Error: {str(e)}")
-

datacontract/integration/publish_opentelemetry.py

@@ -1,21 +1,24 @@
 import logging
+import math
 import os
 from importlib import metadata
-from uuid import uuid4
-import math
 
-from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
-from opentelemetry.metrics import Observation
-
-from datacontract.model.run import \
-    Run
 from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import \
+    OTLPMetricExporter as OTLPgRPCMetricExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import \
+    OTLPMetricExporter
+from opentelemetry.metrics import Observation
 from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, \
+    PeriodicExportingMetricReader
 
-# Publishes metrics of a test run. 
-# Metric contains the values: 
-# 0 == test run passed, 
+from datacontract.model.run import Run
+
+
+# Publishes metrics of a test run.
+# Metric contains the values:
+# 0 == test run passed,
 # 1 == test run has warnings
 # 2 == test run failed
 # 3 == test run not possible due to an error
@@ -26,13 +29,14 @@ from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExpo
 # OTEL_SERVICE_NAME=datacontract-cli
 # OTEL_EXPORTER_OTLP_ENDPOINT=https://YOUR_ID.apm.westeurope.azure.elastic-cloud.com:443
 # OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20secret (Optional, when using SaaS Products)
-# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf (Optional, because it is the default value)
+# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf and OTEL_EXPORTER_OTLP_PROTOCOL=grpc
 #
 # Current limitations:
 # - no gRPC support
 # - currently, only ConsoleExporter and OTLP Exporter
 # - Metrics only, no logs yet (but loosely planned)
 
+
 def publish_opentelemetry(run: Run):
     try:
         if run.dataContractId is None:
@@ -48,7 +52,8 @@ def publish_opentelemetry(run: Run):
             name="datacontract.cli.test",
             callbacks=[lambda x: _to_observation_callback(run)],
             unit="result",
-            description="The overall result of the data contract test run")
+            description="The overall result of the data contract test run",
+        )
 
         telemetry.publish()
     except Exception as e:
@@ -80,10 +85,19 @@ def _to_observation(run):
 
 class Telemetry:
     def __init__(self):
-        self.exporter = ConsoleMetricExporter()
-        self.remote_exporter = OTLPMetricExporter()
+        protocol = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL")
+
+        # lower to allow grpc, GRPC and alike values.
+        if protocol and protocol.lower() == "grpc":
+            self.remote_exporter = OTLPgRPCMetricExporter()
+        else:
+            # Fallback to default OTEL http/protobuf which is used when the variable is not set.
+            # This Exporter also works for http/json.
+            self.remote_exporter = OTLPMetricExporter()
+
+        self.console_exporter = ConsoleMetricExporter()
         # using math.inf so it does not collect periodically. we do this in collect ourselves, one-time.
-        self.reader = PeriodicExportingMetricReader(self.exporter, export_interval_millis=math.inf)
+        self.reader = PeriodicExportingMetricReader(self.console_exporter, export_interval_millis=math.inf)
         self.remote_reader = PeriodicExportingMetricReader(self.remote_exporter, export_interval_millis=math.inf)
         provider = MeterProvider(metric_readers=[self.reader, self.remote_reader])
         metrics.set_meter_provider(provider)

datacontract/lint/files.py

@@ -10,8 +10,8 @@ def read_file(path):
             name=f"Reading data contract from {path}",
             reason=f"The file '{path}' does not exist.",
             engine="datacontract",
-            result="error"
+            result="error",
         )
-    with open(path, 'r') as file:
+    with open(path, "r") as file:
         file_content = file.read()
     return file_content

datacontract/lint/lint.py

@@ -1,10 +1,10 @@
-from enum import Enum
+import abc
 from dataclasses import dataclass, field
+from enum import Enum
 from typing import Sequence, Any, cast
-import abc
 
-from ..model.data_contract_specification import DataContractSpecification
 from datacontract.model.run import Check
+from ..model.data_contract_specification import DataContractSpecification
 
 """This module contains linter definitions for linting a data contract.
 
@@ -17,10 +17,11 @@ contract."""
 
 class LintSeverity(Enum):
     """The severity of a lint message. Generally, lint messages should be
-       emitted with a severity of ERROR. WARNING should be used when the linter
-       cannot determine a lint result, for example, when an unsupported model
-       type is used.
+    emitted with a severity of ERROR. WARNING should be used when the linter
+    cannot determine a lint result, for example, when an unsupported model
+    type is used.
     """
+
     ERROR = 2
     WARNING = 1
 
@@ -36,6 +37,7 @@ class LinterMessage:
        model: The model that caused the lint to fail. Is optional.
 
     """
+
     outcome: LintSeverity
     message: str
     model: Any = None
@@ -60,6 +62,7 @@ class LinterResult:
           results can be present in the list. An empty list means that
           the linter ran without producing warnings or errors.
     """
+
     results: Sequence[LinterMessage] = field(default_factory=list)
 
     @classmethod
@@ -72,34 +75,29 @@ class LinterResult:
 
     def with_warning(self, message, model=None):
         result = LinterMessage.warning(message, model)
-        return LinterResult(cast(list[LinterMessage],self.results) + [result])
+        return LinterResult(cast(list[LinterMessage], self.results) + [result])
 
     def with_error(self, message, model=None):
         result = LinterMessage.error(message, model)
         return LinterResult(cast(list[LinterMessage], self.results) + [result])
 
     def has_errors(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.ERROR,
-                       self.results))
+        return any(map(lambda result: result.outcome == LintSeverity.ERROR, self.results))
 
     def has_warnings(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.WARNING,
-                       self.results))
+        return any(map(lambda result: result.outcome == LintSeverity.WARNING, self.results))
 
     def error_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results
-                if result.outcome == LintSeverity.ERROR]
+        return [result for result in self.results if result.outcome == LintSeverity.ERROR]
 
     def warning_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results
-                if result.outcome == LintSeverity.WARNING]
+        return [result for result in self.results if result.outcome == LintSeverity.WARNING]
 
     def no_errors_or_warnings(self) -> bool:
         return len(self.results) == 0
 
-    def combine(self, other: 'LinterResult') -> 'LinterResult':
-        return LinterResult(cast(list[Any], self.results) +
-                            cast(list[Any], other.results))
+    def combine(self, other: "LinterResult") -> "LinterResult":
+        return LinterResult(cast(list[Any], self.results) + cast(list[Any], other.results))
 
 
 class Linter(abc.ABC):
@@ -124,23 +122,20 @@ class Linter(abc.ABC):
         result = self.lint_implementation(contract)
         checks = []
         if not result.error_results():
-            checks.append(Check(
-                type="lint",
-                name=f"Linter '{self.name}'",
-                result="passed",
-                engine="datacontract"
-            ))
+            checks.append(Check(type="lint", name=f"Linter '{self.name}'", result="passed", engine="datacontract"))
         else:
             # All linter messages are treated as warnings. Severity is
             # currently ignored, but could be used in filtering in the future
             # Linter messages with level WARNING are currently ignored, but might
             # be logged or printed in the future.
             for lint_error in result.error_results():
-                checks.append(Check(
-                    type="lint",
-                    name=f"Linter '{self.name}'",
-                    result="warning",
-                    engine="datacontract",
-                    reason=lint_error.message
-                ))
+                checks.append(
+                    Check(
+                        type="lint",
+                        name=f"Linter '{self.name}'",
+                        result="warning",
+                        engine="datacontract",
+                        reason=lint_error.message,
+                    )
+                )
         return checks