datacontract-cli 0.10.13__py3-none-any.whl → 0.10.14__py3-none-any.whl

datacontract/cli.py

@@ -244,6 +244,10 @@ def import_(
             help="List of table names to import from the DBML file (repeat for multiple table names, leave empty for all tables in the file)."
         ),
     ] = None,
+    iceberg_table: Annotated[
+        Optional[str],
+        typer.Option(help="Table name to assign to the model created from the Iceberg schema."),
+    ] = None,
 ):
     """
     Create a data contract from the given source location. Prints to stdout.
@@ -259,6 +263,7 @@ def import_(
         dbt_model=dbt_model,
         dbml_schema=dbml_schema,
         dbml_table=dbml_table,
+        iceberg_table=iceberg_table,
     )
     console.print(result.to_yaml())
 

datacontract/data_contract.py

@@ -199,7 +199,15 @@ class DataContract:
 
         except DataContractException as e:
             run.checks.append(
-                Check(type=e.type, result=e.result, name=e.name, reason=e.reason, engine=e.engine, details="")
+                Check(
+                    type=e.type,
+                    name=e.name,
+                    result=e.result,
+                    reason=e.reason,
+                    model=e.model,
+                    engine=e.engine,
+                    details="",
+                )
             )
             run.log_error(str(e))
         except Exception as e:

datacontract/engines/soda/connections/kafka.py

@@ -112,17 +112,38 @@ def get_auth_options():
     """Retrieve Kafka authentication options from environment variables."""
     kafka_sasl_username = os.getenv("DATACONTRACT_KAFKA_SASL_USERNAME")
     kafka_sasl_password = os.getenv("DATACONTRACT_KAFKA_SASL_PASSWORD")
+    kafka_sasl_mechanism = os.getenv("DATACONTRACT_KAFKA_SASL_MECHANISM", "PLAIN").upper()
 
-    if kafka_sasl_username is None or kafka_sasl_username == "":
+    # Skip authentication if credentials are not provided
+    if not kafka_sasl_username or not kafka_sasl_password:
         return {}
 
-    return {
-        "kafka.sasl.mechanism": "PLAIN",
-        "kafka.security.protocol": "SASL_SSL",
-        "kafka.sasl.jaas.config": (
+    # SASL mechanisms supported by Kafka
+    jaas_config = {
+        "PLAIN": (
             f"org.apache.kafka.common.security.plain.PlainLoginModule required "
             f'username="{kafka_sasl_username}" password="{kafka_sasl_password}";'
         ),
+        "SCRAM-SHA-256": (
+            f"org.apache.kafka.common.security.scram.ScramLoginModule required "
+            f'username="{kafka_sasl_username}" password="{kafka_sasl_password}";'
+        ),
+        "SCRAM-SHA-512": (
+            f"org.apache.kafka.common.security.scram.ScramLoginModule required "
+            f'username="{kafka_sasl_username}" password="{kafka_sasl_password}";'
+        ),
+        # Add more mechanisms as needed
+    }
+
+    # Validate SASL mechanism
+    if kafka_sasl_mechanism not in jaas_config:
+        raise ValueError(f"Unsupported SASL mechanism: {kafka_sasl_mechanism}")
+
+    # Return config
+    return {
+        "kafka.sasl.mechanism": kafka_sasl_mechanism,
+        "kafka.security.protocol": "SASL_SSL",
+        "kafka.sasl.jaas.config": jaas_config[kafka_sasl_mechanism],
     }
 
 

datacontract/export/avro_converter.py

@@ -81,9 +81,16 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         return "null"
     if field.type in ["string", "varchar", "text"]:
         return "string"
-    elif field.type in ["number", "decimal", "numeric"]:
+    elif field.type in ["number", "numeric"]:
         # https://avro.apache.org/docs/1.11.1/specification/#decimal
         return "bytes"
+    elif field.type in ["decimal"]:
+        typeVal = {"type": "bytes", "logicalType": "decimal"}
+        if field.scale is not None:
+            typeVal["scale"] = field.scale
+        if field.precision is not None:
+            typeVal["precision"] = field.precision
+        return typeVal
     elif field.type in ["float", "double"]:
         return "double"
     elif field.type in ["integer", "int"]:

datacontract/export/avro_idl_converter.py

@@ -64,6 +64,7 @@ class AvroIDLProtocol:
     model_types: list[AvroModelType]
 
 
+# TODO use DATACONTRACT_TYPES from datacontract/model/data_contract_specification.py
 avro_primitive_types = set(
     [
         "string",

datacontract/export/dcs_exporter.py

@@ -0,0 +1,6 @@
+from datacontract.export.exporter import Exporter
+
+
+class DcsExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return data_contract.to_yaml()

datacontract/export/exporter.py

@@ -10,7 +10,7 @@ class Exporter(ABC):
         self.export_format = export_format
 
     @abstractmethod
-    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict | str:
         pass
 
 
@@ -22,6 +22,8 @@ class ExportFormat(str, Enum):
     dbt_sources = "dbt-sources"
     dbt_staging_sql = "dbt-staging-sql"
     odcs = "odcs"
+    odcs_v2 = "odcs_v2"
+    odcs_v3 = "odcs_v3"
     rdf = "rdf"
     avro = "avro"
     protobuf = "protobuf"
@@ -37,6 +39,7 @@ class ExportFormat(str, Enum):
     spark = "spark"
     sqlalchemy = "sqlalchemy"
     data_caterer = "data-caterer"
+    dcs = "dcs"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/export/exporter_factory.py

@@ -99,7 +99,15 @@ exporter_factory.register_lazy_exporter(
 )
 
 exporter_factory.register_lazy_exporter(
-    name=ExportFormat.odcs, module_path="datacontract.export.odcs_converter", class_name="OdcsExporter"
+    name=ExportFormat.odcs_v2, module_path="datacontract.export.odcs_v2_exporter", class_name="OdcsV2Exporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.odcs_v3, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.odcs, module_path="datacontract.export.odcs_v3_exporter", class_name="OdcsV3Exporter"
 )
 
 exporter_factory.register_lazy_exporter(
@@ -155,3 +163,7 @@ exporter_factory.register_lazy_exporter(
     module_path="datacontract.export.sqlalchemy_converter",
     class_name="SQLAlchemyExporter",
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dcs, module_path="datacontract.export.dcs_exporter", class_name="DcsExporter"
+)

datacontract/export/{odcs_converter.py → odcs_v2_exporter.py}

@@ -6,12 +6,12 @@ from datacontract.model.data_contract_specification import DataContractSpecifica
 from datacontract.export.exporter import Exporter
 
 
-class OdcsExporter(Exporter):
+class OdcsV2Exporter(Exporter):
     def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
-        return to_odcs_yaml(data_contract)
+        return to_odcs_v2_yaml(data_contract)
 
 
-def to_odcs_yaml(data_contract_spec: DataContractSpecification):
+def to_odcs_v2_yaml(data_contract_spec: DataContractSpecification):
     odcs = {
         "kind": "DataContract",
         "apiVersion": "2.3.0",
@@ -25,7 +25,7 @@ def to_odcs_yaml(data_contract_spec: DataContractSpecification):
     if data_contract_spec.info.contact is not None:
         if data_contract_spec.info.contact.email is not None:
             odcs["productDl"] = data_contract_spec.info.contact.email
-        if data_contract_spec.info.contact.email is not None:
+        if data_contract_spec.info.contact.url is not None:
             odcs["productFeedbackUrl"] = data_contract_spec.info.contact.url
 
     if data_contract_spec.terms is not None:

datacontract/export/odcs_v3_exporter.py

@@ -0,0 +1,294 @@
+from typing import Dict
+
+import yaml
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+
+
+class OdcsV3Exporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_odcs_v3_yaml(data_contract)
+
+
+def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
+    odcs = {
+        "apiVersion": "v3.0.0",
+        "kind": "DataContract",
+        "id": data_contract_spec.id,
+        "name": data_contract_spec.info.title,
+        "version": data_contract_spec.info.version,
+        "domain": data_contract_spec.info.owner,
+        "status": data_contract_spec.info.status,
+    }
+
+    if data_contract_spec.terms is not None:
+        odcs["description"] = {
+            "purpose": data_contract_spec.terms.description.strip()
+            if data_contract_spec.terms.description is not None
+            else None,
+            "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
+            "limitations": data_contract_spec.terms.limitations.strip()
+            if data_contract_spec.terms.limitations is not None
+            else None,
+        }
+
+    odcs["schema"] = []
+    for model_key, model_value in data_contract_spec.models.items():
+        odcs_schema = to_odcs_schema(model_key, model_value)
+        odcs["schema"].append(odcs_schema)
+
+    if data_contract_spec.servicelevels is not None:
+        slas = []
+        if data_contract_spec.servicelevels.availability is not None:
+            slas.append(
+                {
+                    "property": "generalAvailability",
+                    "value": data_contract_spec.servicelevels.availability.description,
+                }
+            )
+        if data_contract_spec.servicelevels.retention is not None:
+            slas.append({"property": "retention", "value": data_contract_spec.servicelevels.retention.period})
+
+        if len(slas) > 0:
+            odcs["slaProperties"] = slas
+
+    if data_contract_spec.info.contact is not None:
+        support = []
+        if data_contract_spec.info.contact.email is not None:
+            support.append(
+                {
+                    "channel": "email",
+                    "url": "mailto:" + data_contract_spec.info.contact.email,
+                }
+            )
+        if data_contract_spec.info.contact.url is not None:
+            support.append(
+                {
+                    "channel": "other",
+                    "url": data_contract_spec.info.contact.url,
+                }
+            )
+        if len(support) > 0:
+            odcs["support"] = support
+
+    if data_contract_spec.servers is not None and len(data_contract_spec.servers) > 0:
+        servers = []
+
+        for server_key, server_value in data_contract_spec.servers.items():
+            server_dict = {}
+            server_dict["server"] = server_key
+            if server_value.type is not None:
+                server_dict["type"] = server_value.type
+            if server_value.environment is not None:
+                server_dict["environment"] = server_value.environment
+            if server_value.account is not None:
+                server_dict["account"] = server_value.account
+            if server_value.database is not None:
+                server_dict["database"] = server_value.database
+            if server_value.schema_ is not None:
+                server_dict["schema"] = server_value.schema_
+            if server_value.format is not None:
+                server_dict["format"] = server_value.format
+            if server_value.project is not None:
+                server_dict["project"] = server_value.project
+            if server_value.dataset is not None:
+                server_dict["dataset"] = server_value.dataset
+            if server_value.path is not None:
+                server_dict["path"] = server_value.path
+            if server_value.delimiter is not None:
+                server_dict["delimiter"] = server_value.delimiter
+            if server_value.endpointUrl is not None:
+                server_dict["endpointUrl"] = server_value.endpointUrl
+            if server_value.location is not None:
+                server_dict["location"] = server_value.location
+            if server_value.host is not None:
+                server_dict["host"] = server_value.host
+            if server_value.port is not None:
+                server_dict["port"] = server_value.port
+            if server_value.catalog is not None:
+                server_dict["catalog"] = server_value.catalog
+            if server_value.topic is not None:
+                server_dict["topic"] = server_value.topic
+            if server_value.http_path is not None:
+                server_dict["http_path"] = server_value.http_path
+            if server_value.token is not None:
+                server_dict["token"] = server_value.token
+            if server_value.driver is not None:
+                server_dict["driver"] = server_value.driver
+            if server_value.roles is not None:
+                server_dict["roles"] = [
+                    {"name": role.name, "description": role.description} for role in server_value.roles
+                ]
+            servers.append(server_dict)
+
+        if len(servers) > 0:
+            odcs["servers"] = servers
+
+    odcs["customProperties"] = []
+    if data_contract_spec.info.model_extra is not None:
+        for key, value in data_contract_spec.info.model_extra.items():
+            odcs["customProperties"].append({"property": key, "value": value})
+    if len(odcs["customProperties"]) == 0:
+        del odcs["customProperties"]
+
+    return yaml.dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
+
+
+def to_odcs_schema(model_key, model_value: Model) -> dict:
+    odcs_table = {
+        "name": model_key,
+        "physicalName": model_key,
+        "logicalType": "object",
+        "physicalType": model_value.type,
+    }
+    if model_value.description is not None:
+        odcs_table["description"] = model_value.description
+    properties = to_properties(model_value.fields)
+    if properties:
+        odcs_table["properties"] = properties
+
+    odcs_table["customProperties"] = []
+    if model_value.model_extra is not None:
+        for key, value in model_value.model_extra.items():
+            odcs_table["customProperties"].append({"property": key, "value": value})
+    if len(odcs_table["customProperties"]) == 0:
+        del odcs_table["customProperties"]
+
+    return odcs_table
+
+
+def to_properties(fields: Dict[str, Field]) -> list:
+    properties = []
+    for field_name, field in fields.items():
+        property = to_property(field_name, field)
+        properties.append(property)
+    return properties
+
+
+def to_logical_type(type: str) -> str | None:
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "string"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "date"
+    if type.lower() in ["timestamp_ntz"]:
+        return "date"
+    if type.lower() in ["date"]:
+        return "date"
+    if type.lower() in ["time"]:
+        return "string"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        return "number"
+    if type.lower() in ["float", "double"]:
+        return "number"
+    if type.lower() in ["integer", "int", "long", "bigint"]:
+        return "integer"
+    if type.lower() in ["boolean"]:
+        return "boolean"
+    if type.lower() in ["object", "record", "struct"]:
+        return "object"
+    if type.lower() in ["bytes"]:
+        return "array"
+    if type.lower() in ["array"]:
+        return "array"
+    if type.lower() in ["null"]:
+        return None
+    return None
+
+
+def to_physical_type(type: str) -> str | None:
+    # TODO: to we need to do a server mapping here?
+    return type
+
+
+def to_property(field_name: str, field: Field) -> dict:
+    property = {"name": field_name}
+    if field.title is not None:
+        property["businessName"] = field.title
+    if field.type is not None:
+        property["logicalType"] = to_logical_type(field.type)
+        property["physicalType"] = to_physical_type(field.type)
+    if field.description is not None:
+        property["description"] = field.description
+    if field.required is not None:
+        property["isNullable"] = not field.required
+    if field.unique is not None:
+        property["isUnique"] = field.unique
+    if field.classification is not None:
+        property["classification"] = field.classification
+    if field.examples is not None:
+        property["examples"] = field.examples
+    if field.example is not None:
+        property["examples"] = [field.example]
+
+    property["customProperties"] = []
+    if field.model_extra is not None:
+        for key, value in field.model_extra.items():
+            property["customProperties"].append({"property": key, "value": value})
+    if field.pii is not None:
+        property["customProperties"].append({"property": "pii", "value": field.pii})
+    if property.get("customProperties") is not None and len(property["customProperties"]) == 0:
+        del property["customProperties"]
+
+    property["tags"] = []
+    if field.tags is not None:
+        property["tags"].extend(field.tags)
+    if not property["tags"]:
+        del property["tags"]
+
+    property["logicalTypeOptions"] = {}
+    if field.minLength is not None:
+        property["logicalTypeOptions"]["minLength"] = field.minLength
+    if field.maxLength is not None:
+        property["logicalTypeOptions"]["maxLength"] = field.maxLength
+    if field.pattern is not None:
+        property["logicalTypeOptions"]["pattern"] = field.pattern
+    if field.minimum is not None:
+        property["logicalTypeOptions"]["minimum"] = field.minimum
+    if field.maximum is not None:
+        property["logicalTypeOptions"]["maximum"] = field.maximum
+    if field.exclusiveMinimum is not None:
+        property["logicalTypeOptions"]["exclusiveMinimum"] = field.exclusiveMinimum
+    if field.exclusiveMaximum is not None:
+        property["logicalTypeOptions"]["exclusiveMaximum"] = field.exclusiveMaximum
+    if property["logicalTypeOptions"] == {}:
+        del property["logicalTypeOptions"]
+
+    if field.quality is not None:
+        quality_property = []
+        for quality in field.quality:
+            quality_dict = {"type": quality.type}
+            if quality.description is not None:
+                quality_dict["description"] = quality.description
+            if quality.query is not None:
+                quality_dict["query"] = quality.query
+            # dialect is not supported in v3.0.0
+            if quality.mustBe is not None:
+                quality_dict["mustBe"] = quality.mustBe
+            if quality.mustNotBe is not None:
+                quality_dict["mustNotBe"] = quality.mustNotBe
+            if quality.mustBeGreaterThan is not None:
+                quality_dict["mustBeGreaterThan"] = quality.mustBeGreaterThan
+            if quality.mustBeGreaterThanOrEqualTo is not None:
+                quality_dict["mustBeGreaterThanOrEqualTo"] = quality.mustBeGreaterThanOrEqualTo
+            if quality.mustBeLessThan is not None:
+                quality_dict["mustBeLessThan"] = quality.mustBeLessThan
+            if quality.mustBeLessThanOrEqualTo is not None:
+                quality_dict["mustBeLessThanOrEqualTo"] = quality.mustBeLessThanOrEqualTo
+            if quality.mustBeBetween is not None:
+                quality_dict["mustBeBetween"] = quality.mustBeBetween
+            if quality.mustNotBeBetween is not None:
+                quality_dict["mustNotBeBetween"] = quality.mustNotBeBetween
+            if quality.engine is not None:
+                quality_dict["engine"] = quality.engine
+            if quality.implementation is not None:
+                quality_dict["implementation"] = quality.implementation
+            quality_property.append(quality_dict)
+        if len(quality_property) > 0:
+            property["quality"] = quality_property
+
+    # todo enum
+
+    return property

datacontract/export/sodacl_converter.py

@@ -1,8 +1,11 @@
+from typing import List
+from venv import logger
+
 import yaml
 
-from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.export.exporter import Exporter
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Quality
 
 
 class SodaExporter(Exporter):
@@ -58,9 +61,14 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
             checks.append(check_field_regex(field_name, field.pattern, quote_field_name))
         if field.enum is not None and len(field.enum) > 0:
             checks.append(check_field_enum(field_name, field.enum, quote_field_name))
+        if field.quality is not None and len(field.quality) > 0:
+            checks.append(check_quality_list(model_key, field_name, field.quality))
         # TODO references: str = None
         # TODO format
 
+    if model_value.quality is not None and len(model_value.quality) > 0:
+        checks.append(check_quality_list(model_key, None, model_value.quality))
+
     checks_for_model_key = f"checks for {model_key}"
 
     if quote_field_name:
@@ -181,6 +189,78 @@ def check_field_regex(field_name, pattern, quote_field_name: bool = False):
     }
 
 
+def check_quality_list(model_name, field_name, quality_list: List[Quality]):
+    checks = {}
+
+    count = 0
+    for quality in quality_list:
+        if quality.type == "sql":
+            if field_name is None:
+                metric_name = f"{model_name}_{field_name}_quality_sql_{count}"
+            else:
+                metric_name = f"{model_name}_quality_sql_{count}"
+            threshold = to_sodacl_threshold(quality)
+            query = prepare_query(quality, model_name, field_name)
+            if query is None:
+                logger.warning(f"Quality check {metric_name} has no query")
+                continue
+            if threshold is None:
+                logger.warning(f"Quality check {metric_name} has no valid threshold")
+                continue
+            checks[f"{metric_name} {threshold}"] = {f"{metric_name} query": query}
+        count += 1
+
+    return checks
+
+
+def prepare_query(quality: Quality, model_name: str, field_name: str = None) -> str | None:
+    if quality.query is None:
+        return None
+    if quality.query == "":
+        return None
+
+    query = quality.query
+
+    query = query.replace("{model}", model_name)
+    query = query.replace("{table}", model_name)
+
+    if field_name is not None:
+        query = query.replace("{field}", field_name)
+        query = query.replace("{column}", field_name)
+
+    return query
+
+
+def to_sodacl_threshold(quality: Quality) -> str | None:
+    if quality.mustBe is not None:
+        return f"= {quality.mustBe}"
+    if quality.mustNotBe is not None:
+        return f"!= {quality.mustNotBe}"
+    if quality.mustBeGreaterThan is not None:
+        return f"> {quality.mustBeGreaterThan}"
+    if quality.mustBeGreaterThanOrEqualTo is not None:
+        return f">= {quality.mustBeGreaterThanOrEqualTo}"
+    if quality.mustBeLessThan is not None:
+        return f"< {quality.mustBeLessThan}"
+    if quality.mustBeLessThanOrEqualTo is not None:
+        return f"<= {quality.mustBeLessThanOrEqualTo}"
+    if quality.mustBeBetween is not None:
+        if len(quality.mustBeBetween) != 2:
+            logger.warning(
+                f"Quality check has invalid mustBeBetween, must have exactly 2 integers in an array: {quality.mustBeBetween}"
+            )
+            return None
+        return f"between {quality.mustBeBetween[0]} and {quality.mustBeBetween[1]}"
+    if quality.mustNotBeBetween is not None:
+        if len(quality.mustNotBeBetween) != 2:
+            logger.warning(
+                f"Quality check has invalid mustNotBeBetween, must have exactly 2 integers in an array: {quality.mustNotBeBetween}"
+            )
+            return None
+        return f"not between {quality.mustNotBeBetween[0]} and {quality.mustNotBeBetween[1]}"
+    return None
+
+
 def add_quality_checks(sodacl, data_contract_spec):
     if data_contract_spec.quality is None:
         return

datacontract/export/spark_converter.py

@@ -128,7 +128,9 @@ def to_data_type(field: Field) -> types.DataType:
     if field_type in ["string", "varchar", "text"]:
         return types.StringType()
     if field_type in ["number", "decimal", "numeric"]:
-        return types.DecimalType(precision=field.precision, scale=field.scale)
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return types.DecimalType(precision=precision, scale=scale)
     if field_type in ["integer", "int"]:
         return types.IntegerType()
     if field_type == "long":