datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/export/pydantic_converter.py

@@ -2,6 +2,12 @@ import ast
 import typing
 
 import datacontract.model.data_contract_specification as spec
+from datacontract.export.exporter import Exporter
+
+
+class PydanticExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_pydantic_model_str(data_contract)
 
 
 def to_pydantic_model_str(contract: spec.DataContractSpecification) -> str:

datacontract/export/rdf_converter.py

@@ -1,8 +1,15 @@
 from pydantic import BaseModel
-from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
+from rdflib import RDF, BNode, Graph, Literal, Namespace, URIRef
 
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification
+
+
+class RdfExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        self.dict_args = export_args
+        rdf_base = self.dict_args.get("rdf_base")
+        return to_rdf_n3(data_contract_spec=data_contract, base=rdf_base)
 
 
 def is_literal(property_name):
@@ -50,8 +57,8 @@ def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
     else:
         g = Graph(base=Namespace(""))
 
-    dc = Namespace("https://datacontract.com/DataContractSpecification/0.9.2/")
-    dcx = Namespace("https://datacontract.com/DataContractSpecification/0.9.2/Extension/")
+    dc = Namespace("https://datacontract.com/DataContractSpecification/1.2.1/")
+    dcx = Namespace("https://datacontract.com/DataContractSpecification/1.2.1/Extension/")
 
     g.bind("dc", dc)
     g.bind("dcx", dcx)
@@ -141,7 +148,7 @@ def add_info(contract, info, graph, dc, dcx):
     graph.add((bnode_info, dc.version, Literal(info.version)))
 
     # add owner
-    owner = URIRef(info.owner)
+    owner = Literal(info.owner)
     graph.add((bnode_info, dc.owner, owner))
 
     # add contact

datacontract/export/sodacl_converter.py

@@ -1,190 +1,38 @@
 import yaml
 
-from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
-
-
-def to_sodacl_yaml(
-    data_contract_spec: DataContractSpecification, server_type: str = None, check_types: bool = True
-) -> str:
-    try:
-        sodacl = {}
-        for model_key, model_value in data_contract_spec.models.items():
-            k, v = to_checks(model_key, model_value, server_type, check_types)
-            sodacl[k] = v
-        add_quality_checks(sodacl, data_contract_spec)
-        sodacl_yaml_str = yaml.dump(sodacl, default_flow_style=False, sort_keys=False)
-        return sodacl_yaml_str
-    except Exception as e:
-        return f"Error: {e}"
-
-
-def to_checks(model_key, model_value, server_type: str, check_types: bool):
-    checks = []
-    fields = model_value.fields
-
-    quote_field_name = server_type in ["postgres"]
-
-    for field_name, field in fields.items():
-        checks.append(check_field_is_present(field_name))
-        if check_types and field.type is not None:
-            sql_type = convert_to_sql_type(field, server_type)
-            checks.append(check_field_type(field_name, sql_type))
-        if field.required:
-            checks.append(check_field_required(field_name, quote_field_name))
-        if field.unique:
-            checks.append(check_field_unique(field_name, quote_field_name))
-        if field.minLength is not None:
-            checks.append(check_field_min_length(field_name, field.minLength))
-        if field.maxLength is not None:
-            checks.append(check_field_max_length(field_name, field.maxLength))
-        if field.minimum is not None:
-            checks.append(check_field_minimum(field_name, field.minimum))
-        if field.maximum is not None:
-            checks.append(check_field_maximum(field_name, field.maximum))
-        if field.exclusiveMinimum is not None:
-            checks.append(check_field_minimum(field_name, field.exclusiveMinimum))
-            checks.append(check_field_not_equal(field_name, field.exclusiveMinimum))
-        if field.exclusiveMaximum is not None:
-            checks.append(check_field_maximum(field_name, field.exclusiveMaximum))
-            checks.append(check_field_not_equal(field_name, field.exclusiveMaximum))
-        if field.pattern is not None:
-            checks.append(check_field_regex(field_name, field.pattern))
-        if field.enum is not None and len(field.enum) > 0:
-            checks.append(check_field_enum(field_name, field.enum))
-        # TODO references: str = None
-        # TODO format
-
-    return f"checks for {model_key}", checks
-
-
-def check_field_is_present(field_name):
-    return {
-        "schema": {
-            "name": f"Check that field {field_name} is present",
-            "fail": {
-                "when required column missing": [field_name],
-            },
-        }
-    }
-
-
-def check_field_type(field_name: str, type: str):
-    return {
-        "schema": {
-            "name": f"Check that field {field_name} has type {type}",
-            "fail": {"when wrong column type": {field_name: type}},
-        }
-    }
-
-
-def check_field_required(field_name: str, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-
-    return {f"missing_count({field_name}) = 0": {"name": f"Check that required field {field_name} has no null values"}}
-
-
-def check_field_unique(field_name, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"duplicate_count({field_name}) = 0": {"name": f"Check that unique field {field_name} has no duplicate values"}
-    }
-
-
-def check_field_min_length(field_name, min_length, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a min length of {min}",
-            "valid min length": min_length,
-        }
-    }
-
-
-def check_field_max_length(field_name, max_length, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a max length of {max_length}",
-            "valid max length": max_length,
-        }
-    }
-
-
-def check_field_minimum(field_name, minimum, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a minimum of {min}",
-            "valid min": minimum,
-        }
-    }
-
-
-def check_field_maximum(field_name, maximum, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a maximum of {maximum}",
-            "valid max": maximum,
-        }
-    }
-
-
-def check_field_not_equal(field_name, value, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} is not equal to {value}",
-            "invalid values": [value],
-        }
-    }
-
-
-def check_field_enum(field_name, enum, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} only contains enum values {enum}",
-            "valid values": enum,
-        }
-    }
-
-
-def check_field_regex(field_name, pattern, quote_field_name: bool = False):
-    if quote_field_name:
-        field_name = f'"{field_name}"'
-    return {
-        f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} matches regex pattern {pattern}",
-            "valid regex": pattern,
-        }
-    }
-
-
-def add_quality_checks(sodacl, data_contract_spec):
-    if data_contract_spec.quality is None:
-        return
-    if data_contract_spec.quality.type is None:
-        return
-    if data_contract_spec.quality.type.lower() != "sodacl":
-        return
-    if isinstance(data_contract_spec.quality.specification, str):
-        quality_specification = yaml.safe_load(data_contract_spec.quality.specification)
-    else:
-        quality_specification = data_contract_spec.quality.specification
-    for key, checks in quality_specification.items():
-        if key in sodacl:
-            for check in checks:
-                sodacl[key].append(check)
-        else:
-            sodacl[key] = checks
+from datacontract.engines.data_contract_checks import create_checks
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification, Server
+from datacontract.model.run import Run
+
+
+class SodaExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> str:
+        run = Run.create_run()
+        server = get_server(data_contract, server)
+        run.checks.extend(create_checks(data_contract, server))
+        return to_sodacl_yaml(run)
+
+
+def to_sodacl_yaml(run: Run) -> str:
+    sodacl_dict = {}
+    for run_check in run.checks:
+        if run_check.engine != "soda" or run_check.language != "sodacl":
+            continue
+        check_yaml_str = run_check.implementation
+        check_yaml_dict = yaml.safe_load(check_yaml_str)
+        for key, value in check_yaml_dict.items():
+            if key in sodacl_dict:
+                if isinstance(sodacl_dict[key], list) and isinstance(value, list):
+                    sodacl_dict[key].extend(value)
+                else:
+                    sodacl_dict[key].update(value)
+            else:
+                sodacl_dict[key] = value
+    return yaml.dump(sodacl_dict)
+
+
+def get_server(data_contract_specification: DataContractSpecification, server_name: str = None) -> Server | None:
+    if server_name is None:
+        return None
+    return data_contract_specification.servers.get(server_name)

datacontract/export/spark_converter.py

@@ -0,0 +1,245 @@
+import json
+
+from pyspark.sql import types
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Field,
+    Model,
+)
+
+
+class SparkExporter(Exporter):
+    """
+    Exporter class for exporting data contracts to Spark schemas.
+    """
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model,
+        server,
+        sql_server_type,
+        export_args,
+    ) -> dict[str, types.StructType]:
+        """
+        Export the given data contract to Spark schemas.
+
+        Args:
+            data_contract (DataContractSpecification): The data contract specification.
+            model: Not used in this implementation.
+            server: Not used in this implementation.
+            sql_server_type: Not used in this implementation.
+            export_args: Additional arguments for export.
+
+        Returns:
+            dict[str, types.StructType]: A dictionary mapping model names to their corresponding Spark schemas.
+        """
+        return to_spark(data_contract)
+
+
+def to_spark(contract: DataContractSpecification) -> str:
+    """
+    Converts a DataContractSpecification into a Spark schema string.
+
+    Args:
+        contract (DataContractSpecification): The data contract specification containing models.
+
+    Returns:
+        str: A string representation of the Spark schema for each model in the contract.
+    """
+    return "\n\n".join(
+        f"{model_name} = {print_schema(to_spark_schema(model))}" for model_name, model in contract.models.items()
+    )
+
+
+def to_spark_dict(contract: DataContractSpecification) -> dict[str, types.StructType]:
+    """
+    Convert a data contract specification to Spark schemas.
+
+    Args:
+        contract (DataContractSpecification): The data contract specification.
+
+    Returns:
+        dict[str, types.StructType]: A dictionary mapping model names to their corresponding Spark schemas.
+    """
+    return {model_name: to_spark_schema(model) for model_name, model in contract.models.items()}
+
+
+def to_spark_schema(model: Model) -> types.StructType:
+    """
+    Convert a model to a Spark schema.
+
+    Args:
+        model (Model): The model to convert.
+
+    Returns:
+        types.StructType: The corresponding Spark schema.
+    """
+    return to_struct_type(model.fields)
+
+
+def to_struct_type(fields: dict[str, Field]) -> types.StructType:
+    """
+    Convert a dictionary of fields to a Spark StructType.
+
+    Args:
+        fields (dict[str, Field]): The fields to convert.
+
+    Returns:
+        types.StructType: The corresponding Spark StructType.
+    """
+    struct_fields = [to_struct_field(field, field_name) for field_name, field in fields.items()]
+    return types.StructType(struct_fields)
+
+
+def to_struct_field(field: Field, field_name: str) -> types.StructField:
+    """
+    Convert a field to a Spark StructField.
+
+    Args:
+        field (Field): The field to convert.
+        field_name (str): The name of the field.
+
+    Returns:
+        types.StructField: The corresponding Spark StructField.
+    """
+    data_type = to_spark_data_type(field)
+    metadata = to_spark_metadata(field)
+    return types.StructField(name=field_name, dataType=data_type, nullable=not field.required, metadata=metadata)
+
+
+def to_spark_data_type(field: Field) -> types.DataType:
+    """
+    Convert a field to a Spark DataType.
+
+    Args:
+        field (Field): The field to convert.
+
+    Returns:
+        types.DataType: The corresponding Spark DataType.
+    """
+    field_type = field.type
+    if field_type is None or field_type in ["null"]:
+        return types.NullType()
+    if field_type == "array":
+        return types.ArrayType(to_spark_data_type(field.items))
+    if field_type in ["object", "record", "struct"]:
+        return types.StructType(to_struct_type(field.fields))
+    if field_type == "map":
+        return types.MapType(to_spark_data_type(field.keys), to_spark_data_type(field.values))
+    if field_type == "variant":
+        return types.VariantType()
+    if field_type in ["string", "varchar", "text"]:
+        return types.StringType()
+    if field_type in ["number", "decimal", "numeric"]:
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return types.DecimalType(precision=precision, scale=scale)
+    if field_type in ["integer", "int"]:
+        return types.IntegerType()
+    if field_type == "long":
+        return types.LongType()
+    if field_type == "float":
+        return types.FloatType()
+    if field_type == "double":
+        return types.DoubleType()
+    if field_type == "boolean":
+        return types.BooleanType()
+    if field_type in ["timestamp", "timestamp_tz"]:
+        return types.TimestampType()
+    if field_type == "timestamp_ntz":
+        return types.TimestampNTZType()
+    if field_type == "date":
+        return types.DateType()
+    if field_type == "bytes":
+        return types.BinaryType()
+    return types.StringType()  # default if no condition is met
+
+
+def to_spark_metadata(field: Field) -> dict[str, str]:
+    """
+    Convert a field to a Spark metadata dictonary.
+
+    Args:
+        field (Field): The field to convert.
+
+    Returns:
+        dict: dictionary that can be supplied to Spark as metadata for a StructField
+    """
+
+    metadata = {}
+    if field.description:
+        metadata["comment"] = field.description
+
+    return metadata
+
+
+def print_schema(dtype: types.DataType) -> str:
+    """
+    Converts a PySpark DataType schema to its equivalent code representation.
+
+    Args:
+        dtype (types.DataType): The PySpark DataType schema to be converted.
+
+    Returns:
+        str: The code representation of the PySpark DataType schema.
+    """
+
+    def indent(text: str, level: int) -> str:
+        """
+        Indents each line of the given text by a specified number of levels.
+
+        Args:
+            text (str): The text to be indented.
+            level (int): The number of indentation levels.
+
+        Returns:
+            str: The indented text.
+        """
+        return "\n".join([f"{'    ' * level}{line}" for line in text.split("\n")])
+
+    def repr_column(column: types.StructField) -> str:
+        """
+        Converts a PySpark StructField to its code representation.
+
+        Args:
+            column (types.StructField): The StructField to be converted.
+
+        Returns:
+            str: The code representation of the StructField.
+        """
+        name = f'"{column.name}"'
+        data_type = indent(print_schema(column.dataType), 1)
+        nullable = indent(f"{column.nullable}", 1)
+        if column.metadata:
+            metadata = indent(f"{json.dumps(column.metadata)}", 1)
+            return f"StructField({name},\n{data_type},\n{nullable},\n{metadata}\n)"
+        else:
+            return f"StructField({name},\n{data_type},\n{nullable}\n)"
+
+    def format_struct_type(struct_type: types.StructType) -> str:
+        """
+        Converts a PySpark StructType to its code representation.
+
+        Args:
+            struct_type (types.StructType): The StructType to be converted.
+
+        Returns:
+            str: The code representation of the StructType.
+        """
+        fields = ",\n".join([indent(repr_column(field), 1) for field in struct_type.fields])
+        return f"StructType([\n{fields}\n])"
+
+    if isinstance(dtype, types.StructType):
+        return format_struct_type(dtype)
+    elif isinstance(dtype, types.ArrayType):
+        return f"ArrayType({print_schema(dtype.elementType)})"
+    elif isinstance(dtype, types.MapType):
+        return f"MapType(\n{indent(print_schema(dtype.keyType), 1)}, {print_schema(dtype.valueType)})"
+    elif isinstance(dtype, types.DecimalType):
+        return f"DecimalType({dtype.precision}, {dtype.scale})"
+    else:
+        dtype_str = str(dtype)
+        return dtype_str if dtype_str.endswith("()") else f"{dtype_str}()"

datacontract/export/sql_converter.py

@@ -1,7 +1,29 @@
+from datacontract.export.exporter import Exporter, _check_models_for_export, _determine_sql_server_type
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification, Model
 
 
+class SqlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> str:
+        server_type = _determine_sql_server_type(
+            data_contract,
+            sql_server_type,
+        )
+        return to_sql_ddl(data_contract, server_type, export_args.get("server"))
+
+
+class SqlQueryExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> str:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        server_type = _determine_sql_server_type(data_contract, sql_server_type, export_args.get("server"))
+        return to_sql_query(
+            data_contract,
+            model_name,
+            model_value,
+            server_type,
+        )
+
+
 def to_sql_query(
     data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake"
 ) -> str:
@@ -37,7 +59,9 @@ def _to_sql_query(model_name, model_value, server_type) -> str:
     return result
 
 
-def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str = "snowflake") -> str:
+def to_sql_ddl(
+    data_contract_spec: DataContractSpecification, server_type: str = "snowflake", server: str = None
+) -> str:
     if data_contract_spec is None:
         return ""
     if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
@@ -45,7 +69,12 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str =
 
     table_prefix = ""
 
-    for server_name, server in iter(data_contract_spec.servers.items()):
+    if server is None:
+        servers = data_contract_spec.servers
+    else:
+        servers = {server: data_contract_spec.servers[server]}
+
+    for server_name, server in iter(servers.items()):
         if server.type == "snowflake":
             server_type = "snowflake"
             break
@@ -63,6 +92,7 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str =
     result = ""
     result += f"-- Data Contract: {data_contract_spec.id}\n"
     result += f"-- SQL Dialect: {server_type}\n"
+
     for model_name, model in iter(data_contract_spec.models.items()):
         result += _to_sql_table(table_prefix + model_name, model, server_type)
 
@@ -83,10 +113,12 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
         result += f"  {field_name} {type}"
         if field.required:
             result += " not null"
-        if field.primary:
+        if field.primaryKey or field.primary:
             result += " primary key"
         if server_type == "databricks" and field.description is not None:
             result += f' COMMENT "{_escape(field.description)}"'
+        if server_type == "snowflake" and field.description is not None:
+            result += f" COMMENT '{_escape(field.description)}'"
         if current_field_index < fields:
             result += ","
         result += "\n"
@@ -94,6 +126,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
     result += ")"
     if server_type == "databricks" and model.description is not None:
         result += f' COMMENT "{_escape(model.description)}"'
+    if server_type == "snowflake" and model.description is not None:
+        result += f" COMMENT='{_escape(model.description)}'"
     result += ";\n"
     return result