datacontract-cli 0.9.6.post2__py3-none-any.whl → 0.9.8__py3-none-any.whl

datacontract/export/sql_type_converter.py

@@ -0,0 +1,131 @@
+from datacontract.model.data_contract_specification import Field
+
+
+def convert_to_sql_type(field: Field, server_type: str) -> str:
+    if server_type == "snowflake":
+        return convert_to_snowflake(field)
+    if server_type == "postgres":
+        return convert_type_to_postgres(field)
+    if server_type == "databricks":
+        return convert_to_databricks(field)
+    return field.type
+
+
+# snowflake data types:
+# https://docs.snowflake.com/en/sql-reference/data-types.html
+def convert_to_snowflake(field) -> None | str:
+    type = field.type
+    # currently optimized for snowflake
+    # LEARNING: data contract has no direct support for CHAR,CHARACTER
+    # LEARNING: data contract has no support for "date-time", "datetime", "time"
+    # LEARNING: No precision and scale support in data contract
+    # LEARNING: no support for any
+    # GEOGRAPHY and GEOMETRY are not supported by the mapping
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return type.upper()  # STRING, TEXT, VARCHAR are all the same in snowflake
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP_TZ"
+    if type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP_NTZ"
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "TIME"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "NUMBER"
+    if type.lower() in ["float", "double"]:
+        return "FLOAT"
+    if type.lower() in ["integer", "int", "long", "bigint"]:
+        return "NUMBER"  # always NUMBER(38,0)
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        return "OBJECT"
+    if type.lower() in ["bytes"]:
+        return "BINARY"
+    if type.lower() in ["array"]:
+        return "ARRAY"
+    return None
+
+
+# https://www.postgresql.org/docs/current/datatype.html
+# Using the name whenever possible
+def convert_type_to_postgres(field: Field) -> None | str:
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        if field.format == "uuid":
+            return "uuid"
+        return "text"  # STRING does not exist, TEXT and VARCHAR are all the same in postrges
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "timestamptz"
+    if type.lower() in ["timestamp_ntz"]:
+        return "timestamp"
+    if type.lower() in ["date"]:
+        return "date"
+    if type.lower() in ["time"]:
+        return "time"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        if type.lower() == "number":
+            return "numeric"
+        return type.lower()
+    if type.lower() in ["float"]:
+        return "real"
+    if type.lower() in ["double"]:
+        return "double precision"
+    if type.lower() in ["integer", "int", "bigint"]:
+        return type.lower()
+    if type.lower() in ["long"]:
+        return "bigint"
+    if type.lower() in ["boolean"]:
+        return "boolean"
+    if type.lower() in ["object", "record", "struct"]:
+        return "jsonb"
+    if type.lower() in ["bytes"]:
+        return "bytea"
+    if type.lower() in ["array"]:
+        return convert_to_sql_type(field.items, "postgres") + "[]"
+    return None
+
+
+# databricks data types:
+# https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
+def convert_to_databricks(field) -> None | str:
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "STRING"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP_NTZ"
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "STRING"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "DECIMAL"
+    if type.lower() in ["float"]:
+        return "FLOAT"
+    if type.lower() in ["double"]:
+        return "DOUBLE"
+    if type.lower() in ["integer", "int"]:
+        return "INT"
+    if type.lower() in ["long", "bigint"]:
+        return "BIGINT"
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        return "STRUCT"
+    if type.lower() in ["bytes"]:
+        return "BINARY"
+    if type.lower() in ["array"]:
+        return "ARRAY"
+    return None

datacontract/export/terraform_converter.py

@@ -0,0 +1,71 @@
+import re
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Server
+
+
+def to_terraform(data_contract_spec: DataContractSpecification, server_id: str = None) -> str:
+    if data_contract_spec is None:
+        return ""
+    if data_contract_spec.servers is None or len(data_contract_spec.servers) == 0:
+        return ""
+
+    result = ""
+    for server_name, server in iter(data_contract_spec.servers.items()):
+        if server_id is not None and server_name != server_id:
+            continue
+        result = server_to_terraform_resource(data_contract_spec, result, server, server_name)
+
+    return result.strip()
+
+
+def server_to_terraform_resource(data_contract_spec, result, server: Server, server_name):
+    tag_data_contract = data_contract_spec.id
+    tag_name = data_contract_spec.info.title
+    tag_server = server_name
+    bucket_name = extract_bucket_name(server)
+    resource_id = f"{data_contract_spec.id}_{server_name}"
+    data_product_id = server.dataProductId
+
+    if data_product_id is not None:
+        result += f"""
+resource "aws_s3_bucket" "{resource_id}" {{
+  bucket = "{bucket_name}"
+
+  tags = {{
+    Name         = "{tag_name}"
+    DataContract = "{tag_data_contract}"
+    Server       = "{tag_server}"
+    DataProduct  = "{data_product_id}"
+  }}
+}}
+
+"""
+    else:
+        result += f"""
+resource "aws_s3_bucket" "{resource_id}" {{
+  bucket = "{bucket_name}"
+
+  tags = {{
+    Name         = "{tag_name}"
+    DataContract = "{tag_data_contract}"
+    Server       = "{tag_server}"
+  }}
+}}
+
+"""
+    return result
+
+
+def extract_bucket_name(server) -> str | None:
+    if server.type == "s3":
+        s3_url = server.location
+        # Regular expression to match the S3 bucket name
+        match = re.search(r"s3://([^/]+)/", s3_url)
+        if match:
+            # Return the first group (bucket name)
+            return match.group(1)
+        else:
+            return ""
+
+    return ""

datacontract/imports/avro_importer.py

@@ -0,0 +1,106 @@
+import avro.schema
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Model, Field
+from datacontract.model.exceptions import DataContractException
+
+
+def import_avro(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    try:
+        with open(source, "r") as file:
+            avro_schema = avro.schema.parse(file.read())
+    except Exception as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse avro schema",
+            reason=f"Failed to parse avro schema from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    # type record is being used for both the table and the object types in data contract
+    # -> CONSTRAINT: one table per .avsc input, all nested records are interpreted as objects
+    fields = import_record_fields(avro_schema.fields)
+
+    data_contract_specification.models[avro_schema.name] = Model(
+        type="table",
+        fields=fields,
+    )
+
+    if avro_schema.get_prop("doc") is not None:
+        data_contract_specification.models[avro_schema.name].description = avro_schema.get_prop("doc")
+
+    if avro_schema.get_prop("namespace") is not None:
+        data_contract_specification.models[avro_schema.name].namespace = avro_schema.get_prop("namespace")
+
+    return data_contract_specification
+
+
+def import_record_fields(record_fields):
+    imported_fields = {}
+    for field in record_fields:
+
+        imported_fields[field.name] = Field()
+        imported_fields[field.name].required = True
+        imported_fields[field.name].description = field.doc
+
+        if field.type.type == "record":
+            imported_fields[field.name].type = "object"
+            imported_fields[field.name].description = field.type.doc
+            imported_fields[field.name].fields = import_record_fields(field.type.fields)
+        elif field.type.type == "union":
+            imported_fields[field.name].required = False
+            imported_fields[field.name].type = import_type_of_optional_field(field)
+        else: # primitive type
+            imported_fields[field.name].type = map_type_from_avro(field.type.type)
+    return imported_fields
+
+
+def import_type_of_optional_field(field):
+    for field_type in field.type.schemas:
+        if field_type.type != "null":
+            return map_type_from_avro(field_type.type)
+    raise DataContractException(
+        type="schema",
+        result="failed",
+        name="Map avro type to data contract type",
+        reason="Could not import optional field: union type does not contain a non-null type",
+        engine="datacontract",
+    )
+
+
+def map_type_from_avro(avro_type_str: str):
+    # TODO: ambiguous mapping in the export
+    if avro_type_str == "null":
+        return "null"
+    elif avro_type_str == "string":
+        return "string"
+    elif avro_type_str == "bytes":
+        return "binary"
+    elif avro_type_str == "double":
+        return "double"
+    elif avro_type_str == "int":
+        return "int"
+    elif avro_type_str == "long":
+        return "long"
+    elif avro_type_str == "boolean":
+        return "boolean"
+    elif avro_type_str == "array":
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="Map avro type to data contract type",
+            reason="Array type not supported",
+            engine="datacontract",
+        )
+    else:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="Map avro type to data contract type",
+            reason=f"Unsupported type {avro_type_str} in avro schema.",
+            engine="datacontract",
+        )

datacontract/imports/sql_importer.py

@@ -5,12 +5,10 @@ from datacontract.model.data_contract_specification import \
 
 
 def import_sql(data_contract_specification: DataContractSpecification, format: str, source: str):
-
     ddl = parse_from_file(source, group_by_type=True)
     tables = ddl["tables"]
 
     for table in tables:
-
         if data_contract_specification.models is None:
             data_contract_specification.models = {}
 

datacontract/init/download_datacontract_file.py

@@ -9,9 +9,9 @@ def download_datacontract_file(file_path: str, from_url: str, overwrite_file: bo
 
     with requests.get(from_url) as response:
         response.raise_for_status()
-        with open(file_path, 'w') as f:
+        with open(file_path, "w") as f:
             f.write(response.text)
 
 
 class FileExistsException(Exception):
-    pass
+    pass

datacontract/integration/publish_datamesh_manager.py

@@ -3,17 +3,16 @@ import os
 
 import requests
 
-from datacontract.model.run import \
-    Run
+from datacontract.model.run import Run
 
 
 def publish_datamesh_manager(run: Run, publish_url: str):
     try:
         if publish_url is None:
-            url = f"https://api.datamesh-manager.com/api/runs"
+            url = "https://api.datamesh-manager.com/api/runs"
         else:
             url = publish_url
-        datamesh_manager_api_key = os.getenv('DATAMESH_MANAGER_API_KEY')
+        datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
 
         if run.dataContractId is None:
             raise Exception("Cannot publish run results, as data contract ID is unknown")
@@ -21,10 +20,7 @@ def publish_datamesh_manager(run: Run, publish_url: str):
         if datamesh_manager_api_key is None:
             raise Exception("Cannot publish run results, as DATAMESH_MANAGER_API_KEY is not set")
 
-        headers = {
-            'Content-Type': 'application/json',
-            'x-api-key': datamesh_manager_api_key
-        }
+        headers = {"Content-Type": "application/json", "x-api-key": datamesh_manager_api_key}
         request_body = run.model_dump_json()
         # print("Request Body:", request_body)
         response = requests.post(url, data=request_body, headers=headers)
@@ -36,4 +32,3 @@ def publish_datamesh_manager(run: Run, publish_url: str):
         logging.info("Published test results to %s", url)
     except Exception as e:
         logging.error(f"Failed publishing test results. Error: {str(e)}")
-

datacontract/integration/publish_opentelemetry.py

@@ -0,0 +1,107 @@
+import logging
+import math
+import os
+from importlib import metadata
+
+from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import \
+    OTLPMetricExporter as OTLPgRPCMetricExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import \
+    OTLPMetricExporter
+from opentelemetry.metrics import Observation
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, \
+    PeriodicExportingMetricReader
+
+from datacontract.model.run import Run
+
+
+# Publishes metrics of a test run.
+# Metric contains the values:
+# 0 == test run passed,
+# 1 == test run has warnings
+# 2 == test run failed
+# 3 == test run not possible due to an error
+# 4 == test status unknown
+#
+# Tested with these environment variables:
+#
+# OTEL_SERVICE_NAME=datacontract-cli
+# OTEL_EXPORTER_OTLP_ENDPOINT=https://YOUR_ID.apm.westeurope.azure.elastic-cloud.com:443
+# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20secret (Optional, when using SaaS Products)
+# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf and OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+#
+# Current limitations:
+# - no gRPC support
+# - currently, only ConsoleExporter and OTLP Exporter
+# - Metrics only, no logs yet (but loosely planned)
+
+
+def publish_opentelemetry(run: Run):
+    try:
+        if run.dataContractId is None:
+            raise Exception("Cannot publish run results, as data contract ID is unknown")
+
+        endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+        logging.info(f"Publishing test results to opentelemetry at {endpoint}")
+
+        telemetry = Telemetry()
+        provider = metrics.get_meter_provider()
+        meter = provider.get_meter("com.datacontract.cli", metadata.version("datacontract-cli"))
+        meter.create_observable_gauge(
+            name="datacontract.cli.test",
+            callbacks=[lambda x: _to_observation_callback(run)],
+            unit="result",
+            description="The overall result of the data contract test run",
+        )
+
+        telemetry.publish()
+    except Exception as e:
+        logging.error(f"Failed publishing test results. Error: {str(e)}")
+
+
+def _to_observation_callback(run):
+    yield _to_observation(run)
+
+
+def _to_observation(run):
+    attributes = {
+        "datacontract.id": run.dataContractId,
+        "datacontract.version": run.dataContractVersion,
+    }
+
+    if run.result == "passed":
+        result_value = 0  # think of exit codes
+    elif run.result == "warning":
+        result_value = 1
+    elif run.result == "failed":
+        result_value = 2
+    elif run.result == "error":
+        result_value = 3
+    else:
+        result_value = 4
+    return Observation(value=result_value, attributes=attributes)
+
+
+class Telemetry:
+    def __init__(self):
+        protocol = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL")
+
+        # lower to allow grpc, GRPC and alike values.
+        if protocol and protocol.lower() == "grpc":
+            self.remote_exporter = OTLPgRPCMetricExporter()
+        else:
+            # Fallback to default OTEL http/protobuf which is used when the variable is not set.
+            # This Exporter also works for http/json.
+            self.remote_exporter = OTLPMetricExporter()
+
+        self.console_exporter = ConsoleMetricExporter()
+        # using math.inf so it does not collect periodically. we do this in collect ourselves, one-time.
+        self.reader = PeriodicExportingMetricReader(self.console_exporter, export_interval_millis=math.inf)
+        self.remote_reader = PeriodicExportingMetricReader(self.remote_exporter, export_interval_millis=math.inf)
+        provider = MeterProvider(metric_readers=[self.reader, self.remote_reader])
+        metrics.set_meter_provider(provider)
+
+    def publish(self):
+        self.reader.collect()
+        self.remote_reader.collect()

datacontract/lint/files.py

@@ -10,8 +10,8 @@ def read_file(path):
             name=f"Reading data contract from {path}",
             reason=f"The file '{path}' does not exist.",
             engine="datacontract",
-            result="error"
+            result="error",
         )
-    with open(path, 'r') as file:
+    with open(path, "r") as file:
         file_content = file.read()
     return file_content

datacontract/lint/lint.py

@@ -1,10 +1,10 @@
-from enum import Enum
-from dataclasses import dataclass, field
-from typing import Sequence, Any
 import abc
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Sequence, Any, cast
 
-from ..model.data_contract_specification import DataContractSpecification
 from datacontract.model.run import Check
+from ..model.data_contract_specification import DataContractSpecification
 
 """This module contains linter definitions for linting a data contract.
 
@@ -17,10 +17,11 @@ contract."""
 
 class LintSeverity(Enum):
     """The severity of a lint message. Generally, lint messages should be
-       emitted with a severity of ERROR. WARNING should be used when the linter
-       cannot determine a lint result, for example, when an unsupported model
-       type is used.
+    emitted with a severity of ERROR. WARNING should be used when the linter
+    cannot determine a lint result, for example, when an unsupported model
+    type is used.
     """
+
     ERROR = 2
     WARNING = 1
 
@@ -31,11 +32,12 @@ class LinterMessage:
     caused the message.
 
     Attributes:
-       outcome: The outcome of the linting, either ERROR or WARNING.
+       outcome: The outcome of the linting, either ERROR or WARNING. Linting outcomes with level WARNING are discarded for now.
        message: A message describing the error or warning in more detail.
        model: The model that caused the lint to fail. Is optional.
 
     """
+
     outcome: LintSeverity
     message: str
     model: Any = None
@@ -60,40 +62,55 @@ class LinterResult:
           results can be present in the list. An empty list means that
           the linter ran without producing warnings or errors.
     """
+
     results: Sequence[LinterMessage] = field(default_factory=list)
 
+    @classmethod
+    def erroneous(cls, message, model=None):
+        return cls([LinterMessage.error(message, model)])
+
+    @classmethod
+    def cautious(cls, message, model=None):
+        return cls([LinterMessage.warning(message, model)])
+
     def with_warning(self, message, model=None):
         result = LinterMessage.warning(message, model)
-        return LinterResult(self.results + [result])
+        return LinterResult(cast(list[LinterMessage], self.results) + [result])
 
     def with_error(self, message, model=None):
         result = LinterMessage.error(message, model)
-        return LinterResult(self.results + [result])
+        return LinterResult(cast(list[LinterMessage], self.results) + [result])
 
     def has_errors(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.ERROR,
-                       self.results))
+        return any(map(lambda result: result.outcome == LintSeverity.ERROR, self.results))
 
     def has_warnings(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.WARNING,
-                       self.results))
+        return any(map(lambda result: result.outcome == LintSeverity.WARNING, self.results))
 
     def error_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results
-                if result.outcome == LintSeverity.ERROR]
+        return [result for result in self.results if result.outcome == LintSeverity.ERROR]
 
     def warning_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results
-                if result.outcome == LintSeverity.WARNING]
+        return [result for result in self.results if result.outcome == LintSeverity.WARNING]
 
     def no_errors_or_warnings(self) -> bool:
         return len(self.results) == 0
 
+    def combine(self, other: "LinterResult") -> "LinterResult":
+        return LinterResult(cast(list[Any], self.results) + cast(list[Any], other.results))
+
 
 class Linter(abc.ABC):
     @property
     @abc.abstractmethod
     def name(self) -> str:
+        """Human-readable name of the linter."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def id(self) -> str:
+        """A linter ID for configuration (i.e. enabling and disabling)."""
         pass
 
     @abc.abstractmethod
@@ -101,26 +118,24 @@ class Linter(abc.ABC):
         pass
 
     def lint(self, contract: DataContractSpecification) -> list[Check]:
+        """Call with a data contract to get a list of check results from the linter."""
         result = self.lint_implementation(contract)
         checks = []
         if not result.error_results():
-            checks.append(Check(
-                type="lint",
-                name=f"Linter '{self.name()}'",
-                result="passed",
-                engine="datacontract"
-            ))
+            checks.append(Check(type="lint", name=f"Linter '{self.name}'", result="passed", engine="datacontract"))
         else:
             # All linter messages are treated as warnings. Severity is
             # currently ignored, but could be used in filtering in the future
             # Linter messages with level WARNING are currently ignored, but might
             # be logged or printed in the future.
             for lint_error in result.error_results():
-                checks.append(Check(
-                    type="lint",
-                    name=f"Linter '{self.name()}'",
-                    result="warning",
-                    engine="datacontract",
-                    reason=lint_error.message
-                ))
+                checks.append(
+                    Check(
+                        type="lint",
+                        name=f"Linter '{self.name}'",
+                        result="warning",
+                        engine="datacontract",
+                        reason=lint_error.message,
+                    )
+                )
         return checks

datacontract/lint/linters/description_linter.py

@@ -0,0 +1,34 @@
+from datacontract.model.data_contract_specification import DataContractSpecification
+from ..lint import Linter, LinterResult
+
+
+class DescriptionLinter(Linter):
+    """Check for a description on contracts, models, model fields, definitions and examples."""
+
+    @property
+    def name(self) -> str:
+        return "Objects have descriptions"
+
+    @property
+    def id(self) -> str:
+        return "description"
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        result = LinterResult()
+        if not contract.info or not contract.info.description:
+            result = result.with_error("Contract has empty description.")
+        for model_name, model in contract.models.items():
+            if not model.description:
+                result = result.with_error(f"Model '{model_name}' has empty description.")
+            for field_name, field in model.fields.items():
+                if not field.description:
+                    result = result.with_error(
+                        f"Field '{field_name}' in model '{model_name}'" f" has empty description."
+                    )
+        for definition_name, definition in contract.definitions.items():
+            if not definition.description:
+                result = result.with_error(f"Definition '{definition_name}' has empty description.")
+        for index, example in enumerate(contract.examples):
+            if not example.description:
+                result = result.with_error(f"Example {index + 1} has empty description.")
+        return result