datacontract-cli 0.10.24__py3-none-any.whl → 0.10.26__py3-none-any.whl

datacontract/api.py

@@ -162,7 +162,7 @@ async def test(
     server: Annotated[
         str | None,
         Query(
-            example="production",
+            examples=["production"],
             description="The server name to test. Optional, if there is only one server.",
         ),
     ] = None,
@@ -191,7 +191,7 @@ async def lint(
     schema: Annotated[
         str | None,
         Query(
-            example="https://datacontract.com/datacontract.schema.json",
+            examples=["https://datacontract.com/datacontract.schema.json"],
             description="The schema to use for validation. This must be a URL.",
         ),
     ] = None,
@@ -220,7 +220,7 @@ def export(
     server: Annotated[
         str | None,
         Query(
-            example="production",
+            examples=["production"],
             description="The server name to export. Optional, if there is only one server.",
         ),
     ] = None,

datacontract/cli.py

@@ -244,7 +244,7 @@ def import_(
     ] = None,
     source: Annotated[
         Optional[str],
-        typer.Option(help="The path to the file or Glue Database that should be imported."),
+        typer.Option(help="The path to the file that should be imported."),
     ] = None,
     dialect: Annotated[
         Optional[str],

datacontract/engines/soda/connections/kafka.py

@@ -27,6 +27,7 @@ def create_spark_session():
     tmp_dir = tempfile.TemporaryDirectory(prefix="datacontract-cli-spark")
     atexit.register(tmp_dir.cleanup)
 
+    pyspark_version = "3.5.5"  # MUST be the same as in the pyproject.toml
     spark = (
         SparkSession.builder.appName("datacontract")
         .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse")
@@ -34,7 +35,7 @@ def create_spark_session():
         .config("spark.ui.enabled", "false")
         .config(
             "spark.jars.packages",
-            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5,org.apache.spark:spark-avro_2.12:3.5.5",
+            f"org.apache.spark:spark-sql-kafka-0-10_2.12:{pyspark_version},org.apache.spark:spark-avro_2.12:{pyspark_version}",
         )
         .getOrCreate()
     )

datacontract/export/great_expectations_converter.py

@@ -19,6 +19,7 @@ from datacontract.export.spark_converter import to_spark_data_type
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import (
     DataContractSpecification,
+    DeprecatedQuality,
     Field,
     Quality,
 )
@@ -91,8 +92,14 @@ def to_great_expectations(
             model_key=model_key, contract_version=data_contract_spec.info.version
         )
     model_value = data_contract_spec.models.get(model_key)
-    quality_checks = get_quality_checks(data_contract_spec.quality)
+
+    # Support for Deprecated Quality
+    quality_checks = get_deprecated_quality_checks(data_contract_spec.quality)
+
+    expectations.extend(get_quality_checks(model_value.quality))
+
     expectations.extend(model_to_expectations(model_value.fields, engine, sql_server_type))
+
     expectations.extend(checks_to_expectations(quality_checks, model_key))
     model_expectation_suite = to_suite(expectations, expectation_suite_name)
 
@@ -135,6 +142,7 @@ def model_to_expectations(fields: Dict[str, Field], engine: str | None, sql_serv
     add_column_order_exp(fields, expectations)
     for field_name, field in fields.items():
         add_field_expectations(field_name, field, expectations, engine, sql_server_type)
+        expectations.extend(get_quality_checks(field.quality, field_name))
     return expectations
 
 
@@ -173,6 +181,8 @@ def add_field_expectations(
         expectations.append(to_column_length_exp(field_name, field.minLength, field.maxLength))
     if field.minimum is not None or field.maximum is not None:
         expectations.append(to_column_min_max_exp(field_name, field.minimum, field.maximum))
+    if field.enum is not None and len(field.enum) != 0:
+        expectations.append(to_column_enum_exp(field_name, field.enum))
 
     return expectations
 
@@ -266,7 +276,24 @@ def to_column_min_max_exp(field_name, minimum, maximum) -> Dict[str, Any]:
     }
 
 
-def get_quality_checks(quality: Quality) -> Dict[str, Any]:
+def to_column_enum_exp(field_name, enum_list: List[str]) -> Dict[str, Any]:
+    """Creates a expect_column_values_to_be_in_set expectation.
+
+    Args:
+        field_name (str): The name of the field.
+        enum_list (Set[str]): enum list of value.
+
+    Returns:
+        Dict[str, Any]: Column value in set expectation.
+    """
+    return {
+        "expectation_type": "expect_column_values_to_be_in_set",
+        "kwargs": {"column": field_name, "value_set": enum_list},
+        "meta": {},
+    }
+
+
+def get_deprecated_quality_checks(quality: DeprecatedQuality) -> Dict[str, Any]:
     """Retrieves quality checks defined in a data contract.
 
     Args:
@@ -288,6 +315,26 @@ def get_quality_checks(quality: Quality) -> Dict[str, Any]:
     return quality_specification
 
 
+def get_quality_checks(qualities: List[Quality], field_name: str | None = None) -> List[Dict[str, Any]]:
+    """Retrieves quality checks defined in a data contract.
+
+    Args:
+        qualities (List[Quality]): List of quality object from the model specification.
+        field_name (str | None): field name if the quality list is attached to a specific field
+
+    Returns:
+        Dict[str, Any]: Dictionary of quality checks.
+    """
+    quality_specification = []
+    for quality in qualities:
+        if quality is not None and quality.engine is not None and quality.engine.lower() == "great-expectations":
+            ge_expectation = quality.implementation
+            if field_name is not None:
+                ge_expectation["column"] = field_name
+            quality_specification.append(ge_expectation)
+    return quality_specification
+
+
 def checks_to_expectations(quality_checks: Dict[str, Any], model_key: str) -> List[Dict[str, Any]]:
     """Converts quality checks to a list of expectations.
 

datacontract/export/odcs_v3_exporter.py

@@ -1,6 +1,17 @@
-from typing import Dict
-
-import yaml
+from typing import Any, Dict
+
+from open_data_contract_standard.model import (
+    CustomProperty,
+    DataQuality,
+    Description,
+    OpenDataContractStandard,
+    Role,
+    SchemaObject,
+    SchemaProperty,
+    Server,
+    ServiceLevelAgreementProperty,
+    Support,
+)
 
 from datacontract.export.exporter import Exporter
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
@@ -12,155 +23,148 @@ class OdcsV3Exporter(Exporter):
 
 
 def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
-    odcs = {
-        "apiVersion": "v3.0.1",
-        "kind": "DataContract",
-        "id": data_contract_spec.id,
-        "name": data_contract_spec.info.title,
-        "version": data_contract_spec.info.version,
-        "status": to_status(data_contract_spec.info.status),
-    }
+    result = OpenDataContractStandard(
+        apiVersion="v3.0.1",
+        kind="DataContract",
+        id=data_contract_spec.id,
+        name=data_contract_spec.info.title,
+        version=data_contract_spec.info.version,
+        status=to_status(data_contract_spec.info.status),
+    )
 
     if data_contract_spec.terms is not None:
-        odcs["description"] = {
-            "purpose": data_contract_spec.terms.description.strip()
+        result.description = Description(
+            purpose=data_contract_spec.terms.description.strip()
             if data_contract_spec.terms.description is not None
             else None,
-            "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
-            "limitations": data_contract_spec.terms.limitations.strip()
+            usage=data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
+            limitations=data_contract_spec.terms.limitations.strip()
             if data_contract_spec.terms.limitations is not None
             else None,
-        }
+        )
 
-    odcs["schema"] = []
+    result.schema_ = []
     for model_key, model_value in data_contract_spec.models.items():
         odcs_schema = to_odcs_schema(model_key, model_value)
-        odcs["schema"].append(odcs_schema)
+        result.schema_.append(odcs_schema)
 
     if data_contract_spec.servicelevels is not None:
         slas = []
         if data_contract_spec.servicelevels.availability is not None:
             slas.append(
-                {
-                    "property": "generalAvailability",
-                    "value": data_contract_spec.servicelevels.availability.description,
-                }
+                ServiceLevelAgreementProperty(
+                    property="generalAvailability", value=data_contract_spec.servicelevels.availability.description
+                )
             )
         if data_contract_spec.servicelevels.retention is not None:
-            slas.append({"property": "retention", "value": data_contract_spec.servicelevels.retention.period})
+            slas.append(
+                ServiceLevelAgreementProperty(
+                    property="retention", value=data_contract_spec.servicelevels.retention.period
+                )
+            )
 
         if len(slas) > 0:
-            odcs["slaProperties"] = slas
+            result.slaProperties = slas
 
     if data_contract_spec.info.contact is not None:
         support = []
         if data_contract_spec.info.contact.email is not None:
-            support.append(
-                {
-                    "channel": "email",
-                    "url": "mailto:" + data_contract_spec.info.contact.email,
-                }
-            )
+            support.append(Support(channel="email", url="mailto:" + data_contract_spec.info.contact.email))
         if data_contract_spec.info.contact.url is not None:
-            support.append(
-                {
-                    "channel": "other",
-                    "url": data_contract_spec.info.contact.url,
-                }
-            )
+            support.append(Support(channel="other", url=data_contract_spec.info.contact.url))
         if len(support) > 0:
-            odcs["support"] = support
+            result.support = support
 
     if data_contract_spec.servers is not None and len(data_contract_spec.servers) > 0:
         servers = []
 
         for server_key, server_value in data_contract_spec.servers.items():
-            server_dict = {}
-            server_dict["server"] = server_key
-            if server_value.type is not None:
-                server_dict["type"] = server_value.type
+            server = Server(server=server_key, type=server_value.type or "")
+
+            # Set all the attributes that are not None
             if server_value.environment is not None:
-                server_dict["environment"] = server_value.environment
+                server.environment = server_value.environment
             if server_value.account is not None:
-                server_dict["account"] = server_value.account
+                server.account = server_value.account
             if server_value.database is not None:
-                server_dict["database"] = server_value.database
+                server.database = server_value.database
             if server_value.schema_ is not None:
-                server_dict["schema"] = server_value.schema_
+                server.schema_ = server_value.schema_
             if server_value.format is not None:
-                server_dict["format"] = server_value.format
+                server.format = server_value.format
             if server_value.project is not None:
-                server_dict["project"] = server_value.project
+                server.project = server_value.project
             if server_value.dataset is not None:
-                server_dict["dataset"] = server_value.dataset
+                server.dataset = server_value.dataset
             if server_value.path is not None:
-                server_dict["path"] = server_value.path
+                server.path = server_value.path
             if server_value.delimiter is not None:
-                server_dict["delimiter"] = server_value.delimiter
+                server.delimiter = server_value.delimiter
             if server_value.endpointUrl is not None:
-                server_dict["endpointUrl"] = server_value.endpointUrl
+                server.endpointUrl = server_value.endpointUrl
             if server_value.location is not None:
-                server_dict["location"] = server_value.location
+                server.location = server_value.location
             if server_value.host is not None:
-                server_dict["host"] = server_value.host
+                server.host = server_value.host
             if server_value.port is not None:
-                server_dict["port"] = server_value.port
+                server.port = server_value.port
             if server_value.catalog is not None:
-                server_dict["catalog"] = server_value.catalog
+                server.catalog = server_value.catalog
             if server_value.topic is not None:
-                server_dict["topic"] = server_value.topic
+                server.topic = server_value.topic
             if server_value.http_path is not None:
-                server_dict["http_path"] = server_value.http_path
+                server.http_path = server_value.http_path
             if server_value.token is not None:
-                server_dict["token"] = server_value.token
+                server.token = server_value.token
             if server_value.driver is not None:
-                server_dict["driver"] = server_value.driver
+                server.driver = server_value.driver
+
             if server_value.roles is not None:
-                server_dict["roles"] = [
-                    {"name": role.name, "description": role.description} for role in server_value.roles
-                ]
-            servers.append(server_dict)
+                server.roles = [Role(role=role.name, description=role.description) for role in server_value.roles]
+
+            servers.append(server)
 
         if len(servers) > 0:
-            odcs["servers"] = servers
+            result.servers = servers
 
-    odcs["customProperties"] = []
+    custom_properties = []
     if data_contract_spec.info.owner is not None:
-        odcs["customProperties"].append({"property": "owner", "value": data_contract_spec.info.owner})
+        custom_properties.append(CustomProperty(property="owner", value=data_contract_spec.info.owner))
     if data_contract_spec.info.model_extra is not None:
         for key, value in data_contract_spec.info.model_extra.items():
-            odcs["customProperties"].append({"property": key, "value": value})
-    if len(odcs["customProperties"]) == 0:
-        del odcs["customProperties"]
+            custom_properties.append(CustomProperty(property=key, value=value))
 
-    return yaml.safe_dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
+    if len(custom_properties) > 0:
+        result.customProperties = custom_properties
 
+    return result.to_yaml()
+
+
+def to_odcs_schema(model_key, model_value: Model) -> SchemaObject:
+    schema_obj = SchemaObject(
+        name=model_key, physicalName=model_key, logicalType="object", physicalType=model_value.type
+    )
 
-def to_odcs_schema(model_key, model_value: Model) -> dict:
-    odcs_table = {
-        "name": model_key,
-        "physicalName": model_key,
-        "logicalType": "object",
-        "physicalType": model_value.type,
-    }
     if model_value.description is not None:
-        odcs_table["description"] = model_value.description
+        schema_obj.description = model_value.description
+
     properties = to_properties(model_value.fields)
     if properties:
-        odcs_table["properties"] = properties
+        schema_obj.properties = properties
 
     model_quality = to_odcs_quality_list(model_value.quality)
     if len(model_quality) > 0:
-        odcs_table["quality"] = model_quality
+        schema_obj.quality = model_quality
 
-    odcs_table["customProperties"] = []
+    custom_properties = []
     if model_value.model_extra is not None:
         for key, value in model_value.model_extra.items():
-            odcs_table["customProperties"].append({"property": key, "value": value})
-    if len(odcs_table["customProperties"]) == 0:
-        del odcs_table["customProperties"]
+            custom_properties.append(CustomProperty(property=key, value=value))
+
+    if len(custom_properties) > 0:
+        schema_obj.customProperties = custom_properties
 
-    return odcs_table
+    return schema_obj
 
 
 def to_properties(fields: Dict[str, Field]) -> list:
@@ -198,82 +202,119 @@ def to_logical_type(type: str) -> str | None:
         return "array"
     if type.lower() in ["array"]:
         return "array"
+    if type.lower() in ["variant"]:
+        return "variant"
     if type.lower() in ["null"]:
         return None
     return None
 
 
-def to_physical_type(type: str) -> str | None:
-    # TODO: to we need to do a server mapping here?
-    return type
+def to_physical_type(config: Dict[str, Any]) -> str | None:
+    if config is None:
+        return None
+    if "postgresType" in config:
+        return config["postgresType"]
+    elif "bigqueryType" in config:
+        return config["bigqueryType"]
+    elif "snowflakeType" in config:
+        return config["snowflakeType"]
+    elif "redshiftType" in config:
+        return config["redshiftType"]
+    elif "sqlserverType" in config:
+        return config["sqlserverType"]
+    elif "databricksType" in config:
+        return config["databricksType"]
+    elif "physicalType" in config:
+        return config["physicalType"]
+    return None
+
+
+def to_property(field_name: str, field: Field) -> SchemaProperty:
+    property = SchemaProperty(name=field_name)
 
+    if field.fields:
+        properties = []
+        for field_name_, field_ in field.fields.items():
+            property_ = to_property(field_name_, field_)
+            properties.append(property_)
+        property.properties = properties
+
+    if field.items:
+        items = to_property(field_name, field.items)
+        items.name = None  # Clear the name for items
+        property.items = items
 
-def to_property(field_name: str, field: Field) -> dict:
-    property = {"name": field_name}
     if field.title is not None:
-        property["businessName"] = field.title
+        property.businessName = field.title
+
     if field.type is not None:
-        property["logicalType"] = to_logical_type(field.type)
-        property["physicalType"] = to_physical_type(field.type)
+        property.logicalType = to_logical_type(field.type)
+        property.physicalType = to_physical_type(field.config)
+
     if field.description is not None:
-        property["description"] = field.description
+        property.description = field.description
+
     if field.required is not None:
-        property["required"] = field.required
+        property.required = field.required
+
     if field.unique is not None:
-        property["unique"] = field.unique
+        property.unique = field.unique
+
     if field.classification is not None:
-        property["classification"] = field.classification
+        property.classification = field.classification
+
     if field.examples is not None:
-        property["examples"] = field.examples.copy()
+        property.examples = field.examples.copy()
+
     if field.example is not None:
-        property["examples"] = [field.example]
+        property.examples = [field.example]
+
     if field.primaryKey is not None and field.primaryKey:
-        property["primaryKey"] = field.primaryKey
-        property["primaryKeyPosition"] = 1
+        property.primaryKey = field.primaryKey
+        property.primaryKeyPosition = 1
+
     if field.primary is not None and field.primary:
-        property["primaryKey"] = field.primary
-        property["primaryKeyPosition"] = 1
+        property.primaryKey = field.primary
+        property.primaryKeyPosition = 1
 
-    property["customProperties"] = []
+    custom_properties = []
     if field.model_extra is not None:
         for key, value in field.model_extra.items():
-            property["customProperties"].append({"property": key, "value": value})
+            custom_properties.append(CustomProperty(property=key, value=value))
+
     if field.pii is not None:
-        property["customProperties"].append({"property": "pii", "value": field.pii})
-    if property.get("customProperties") is not None and len(property["customProperties"]) == 0:
-        del property["customProperties"]
+        custom_properties.append(CustomProperty(property="pii", value=field.pii))
+
+    if len(custom_properties) > 0:
+        property.customProperties = custom_properties
 
-    property["tags"] = []
-    if field.tags is not None:
-        property["tags"].extend(field.tags)
-    if not property["tags"]:
-        del property["tags"]
+    if field.tags is not None and len(field.tags) > 0:
+        property.tags = field.tags
 
-    property["logicalTypeOptions"] = {}
+    logical_type_options = {}
     if field.minLength is not None:
-        property["logicalTypeOptions"]["minLength"] = field.minLength
+        logical_type_options["minLength"] = field.minLength
     if field.maxLength is not None:
-        property["logicalTypeOptions"]["maxLength"] = field.maxLength
+        logical_type_options["maxLength"] = field.maxLength
     if field.pattern is not None:
-        property["logicalTypeOptions"]["pattern"] = field.pattern
+        logical_type_options["pattern"] = field.pattern
     if field.minimum is not None:
-        property["logicalTypeOptions"]["minimum"] = field.minimum
+        logical_type_options["minimum"] = field.minimum
     if field.maximum is not None:
-        property["logicalTypeOptions"]["maximum"] = field.maximum
+        logical_type_options["maximum"] = field.maximum
     if field.exclusiveMinimum is not None:
-        property["logicalTypeOptions"]["exclusiveMinimum"] = field.exclusiveMinimum
+        logical_type_options["exclusiveMinimum"] = field.exclusiveMinimum
     if field.exclusiveMaximum is not None:
-        property["logicalTypeOptions"]["exclusiveMaximum"] = field.exclusiveMaximum
-    if property["logicalTypeOptions"] == {}:
-        del property["logicalTypeOptions"]
+        logical_type_options["exclusiveMaximum"] = field.exclusiveMaximum
+
+    if logical_type_options:
+        property.logicalTypeOptions = logical_type_options
 
     if field.quality is not None:
         quality_list = field.quality
         quality_property = to_odcs_quality_list(quality_list)
         if len(quality_property) > 0:
-            property["quality"] = quality_property
-
-    # todo enum
+            property.quality = quality_property
 
     return property
 
@@ -286,33 +327,35 @@ def to_odcs_quality_list(quality_list):
 
 
 def to_odcs_quality(quality):
-    quality_dict = {"type": quality.type}
+    quality_obj = DataQuality(type=quality.type)
+
     if quality.description is not None:
-        quality_dict["description"] = quality.description
+        quality_obj.description = quality.description
     if quality.query is not None:
-        quality_dict["query"] = quality.query
+        quality_obj.query = quality.query
     # dialect is not supported in v3.0.0
     if quality.mustBe is not None:
-        quality_dict["mustBe"] = quality.mustBe
+        quality_obj.mustBe = quality.mustBe
     if quality.mustNotBe is not None:
-        quality_dict["mustNotBe"] = quality.mustNotBe
+        quality_obj.mustNotBe = quality.mustNotBe
     if quality.mustBeGreaterThan is not None:
-        quality_dict["mustBeGreaterThan"] = quality.mustBeGreaterThan
+        quality_obj.mustBeGreaterThan = quality.mustBeGreaterThan
     if quality.mustBeGreaterThanOrEqualTo is not None:
-        quality_dict["mustBeGreaterThanOrEqualTo"] = quality.mustBeGreaterThanOrEqualTo
+        quality_obj.mustBeGreaterOrEqualTo = quality.mustBeGreaterThanOrEqualTo
     if quality.mustBeLessThan is not None:
-        quality_dict["mustBeLessThan"] = quality.mustBeLessThan
+        quality_obj.mustBeLessThan = quality.mustBeLessThan
     if quality.mustBeLessThanOrEqualTo is not None:
-        quality_dict["mustBeLessThanOrEqualTo"] = quality.mustBeLessThanOrEqualTo
+        quality_obj.mustBeLessOrEqualTo = quality.mustBeLessThanOrEqualTo
     if quality.mustBeBetween is not None:
-        quality_dict["mustBeBetween"] = quality.mustBeBetween
+        quality_obj.mustBeBetween = quality.mustBeBetween
     if quality.mustNotBeBetween is not None:
-        quality_dict["mustNotBeBetween"] = quality.mustNotBeBetween
+        quality_obj.mustNotBeBetween = quality.mustNotBeBetween
     if quality.engine is not None:
-        quality_dict["engine"] = quality.engine
+        quality_obj.engine = quality.engine
     if quality.implementation is not None:
-        quality_dict["implementation"] = quality.implementation
-    return quality_dict
+        quality_obj.implementation = quality.implementation
+
+    return quality_obj
 
 
 def to_status(status):

datacontract/export/spark_converter.py

@@ -175,7 +175,7 @@ def print_schema(dtype: types.DataType) -> str:
         Returns:
             str: The indented text.
         """
-        return "\n".join([f'{"    " * level}{line}' for line in text.split("\n")])
+        return "\n".join([f"{'    ' * level}{line}" for line in text.split("\n")])
 
     def repr_column(column: types.StructField) -> str:
         """

datacontract/export/sql_converter.py

@@ -117,6 +117,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
             result += " primary key"
         if server_type == "databricks" and field.description is not None:
             result += f' COMMENT "{_escape(field.description)}"'
+        if server_type == "snowflake" and field.description is not None:
+            result += f" COMMENT '{_escape(field.description)}'"
         if current_field_index < fields:
             result += ","
         result += "\n"
@@ -124,6 +126,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
     result += ")"
     if server_type == "databricks" and model.description is not None:
         result += f' COMMENT "{_escape(model.description)}"'
+    if server_type == "snowflake" and model.description is not None:
+        result += f" COMMENT='{_escape(model.description)}'"
     result += ";\n"
     return result
 

datacontract/export/sql_type_converter.py

@@ -197,6 +197,8 @@ def convert_to_databricks(field: Field) -> None | str:
     if type.lower() in ["array"]:
         item_type = convert_to_databricks(field.items)
         return f"ARRAY<{item_type}>"
+    if type.lower() in ["variant"]:
+        return "VARIANT"
     return None