datacontract-cli 0.10.7__py3-none-any.whl → 0.10.9__py3-none-any.whl

datacontract/export/sodacl_converter.py

@@ -2,6 +2,12 @@ import yaml
 
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.export.exporter import Exporter
+
+
+class SodaExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_sodacl_yaml(data_contract)
 
 
 def to_sodacl_yaml(
@@ -35,27 +41,32 @@ def to_checks(model_key, model_value, server_type: str, check_types: bool):
         if field.unique:
             checks.append(check_field_unique(field_name, quote_field_name))
         if field.minLength is not None:
-            checks.append(check_field_min_length(field_name, field.minLength))
+            checks.append(check_field_min_length(field_name, field.minLength, quote_field_name))
         if field.maxLength is not None:
-            checks.append(check_field_max_length(field_name, field.maxLength))
+            checks.append(check_field_max_length(field_name, field.maxLength, quote_field_name))
         if field.minimum is not None:
-            checks.append(check_field_minimum(field_name, field.minimum))
+            checks.append(check_field_minimum(field_name, field.minimum, quote_field_name))
         if field.maximum is not None:
-            checks.append(check_field_maximum(field_name, field.maximum))
+            checks.append(check_field_maximum(field_name, field.maximum, quote_field_name))
         if field.exclusiveMinimum is not None:
-            checks.append(check_field_minimum(field_name, field.exclusiveMinimum))
-            checks.append(check_field_not_equal(field_name, field.exclusiveMinimum))
+            checks.append(check_field_minimum(field_name, field.exclusiveMinimum, quote_field_name))
+            checks.append(check_field_not_equal(field_name, field.exclusiveMinimum, quote_field_name))
         if field.exclusiveMaximum is not None:
-            checks.append(check_field_maximum(field_name, field.exclusiveMaximum))
-            checks.append(check_field_not_equal(field_name, field.exclusiveMaximum))
+            checks.append(check_field_maximum(field_name, field.exclusiveMaximum, quote_field_name))
+            checks.append(check_field_not_equal(field_name, field.exclusiveMaximum, quote_field_name))
         if field.pattern is not None:
-            checks.append(check_field_regex(field_name, field.pattern))
+            checks.append(check_field_regex(field_name, field.pattern, quote_field_name))
         if field.enum is not None and len(field.enum) > 0:
-            checks.append(check_field_enum(field_name, field.enum))
+            checks.append(check_field_enum(field_name, field.enum, quote_field_name))
         # TODO references: str = None
         # TODO format
 
-    return f"checks for {model_key}", checks
+    checks_for_model_key = f"checks for {model_key}"
+
+    if quote_field_name:
+        checks_for_model_key = f'checks for "{model_key}"'
+
+    return checks_for_model_key, checks
 
 
 def check_field_is_present(field_name):
@@ -98,7 +109,7 @@ def check_field_min_length(field_name, min_length, quote_field_name: bool = Fals
         field_name = f'"{field_name}"'
     return {
         f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a min length of {min}",
+            "name": f"Check that field {field_name} has a min length of {min_length}",
             "valid min length": min_length,
         }
     }

datacontract/export/spark_converter.py

@@ -0,0 +1,211 @@
+from pyspark.sql import types
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Model,
+    Field,
+)
+from datacontract.export.exporter import Exporter
+
+
+class SparkExporter(Exporter):
+    """
+    Exporter class for exporting data contracts to Spark schemas.
+    """
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model,
+        server,
+        sql_server_type,
+        export_args,
+    ) -> dict[str, types.StructType]:
+        """
+        Export the given data contract to Spark schemas.
+
+        Args:
+            data_contract (DataContractSpecification): The data contract specification.
+            model: Not used in this implementation.
+            server: Not used in this implementation.
+            sql_server_type: Not used in this implementation.
+            export_args: Additional arguments for export.
+
+        Returns:
+            dict[str, types.StructType]: A dictionary mapping model names to their corresponding Spark schemas.
+        """
+        return to_spark(data_contract)
+
+
+def to_spark(contract: DataContractSpecification) -> str:
+    """
+    Converts a DataContractSpecification into a Spark schema string.
+
+    Args:
+        contract (DataContractSpecification): The data contract specification containing models.
+
+    Returns:
+        str: A string representation of the Spark schema for each model in the contract.
+    """
+    return "\n\n".join(
+        f"{model_name} = {print_schema(to_spark_schema(model))}" for model_name, model in contract.models.items()
+    )
+
+
+def to_spark_dict(contract: DataContractSpecification) -> dict[str, types.StructType]:
+    """
+    Convert a data contract specification to Spark schemas.
+
+    Args:
+        contract (DataContractSpecification): The data contract specification.
+
+    Returns:
+        dict[str, types.StructType]: A dictionary mapping model names to their corresponding Spark schemas.
+    """
+    return {model_name: to_spark_schema(model) for model_name, model in contract.models.items()}
+
+
+def to_spark_schema(model: Model) -> types.StructType:
+    """
+    Convert a model to a Spark schema.
+
+    Args:
+        model (Model): The model to convert.
+
+    Returns:
+        types.StructType: The corresponding Spark schema.
+    """
+    return to_struct_type(model.fields)
+
+
+def to_struct_type(fields: dict[str, Field]) -> types.StructType:
+    """
+    Convert a dictionary of fields to a Spark StructType.
+
+    Args:
+        fields (dict[str, Field]): The fields to convert.
+
+    Returns:
+        types.StructType: The corresponding Spark StructType.
+    """
+    struct_fields = [to_struct_field(field, field_name) for field_name, field in fields.items()]
+    return types.StructType(struct_fields)
+
+
+def to_struct_field(field: Field, field_name: str) -> types.StructField:
+    """
+    Convert a field to a Spark StructField.
+
+    Args:
+        field (Field): The field to convert.
+        field_name (str): The name of the field.
+
+    Returns:
+        types.StructField: The corresponding Spark StructField.
+    """
+    data_type = to_data_type(field)
+    return types.StructField(name=field_name, dataType=data_type, nullable=not field.required)
+
+
+def to_data_type(field: Field) -> types.DataType:
+    """
+    Convert a field to a Spark DataType.
+
+    Args:
+        field (Field): The field to convert.
+
+    Returns:
+        types.DataType: The corresponding Spark DataType.
+    """
+    field_type = field.type
+    if field_type is None or field_type in ["null"]:
+        return types.NullType()
+    if field_type == "array":
+        return types.ArrayType(to_data_type(field.items))
+    if field_type in ["object", "record", "struct"]:
+        return types.StructType(to_struct_type(field.fields))
+    if field_type in ["string", "varchar", "text"]:
+        return types.StringType()
+    if field_type in ["number", "decimal", "numeric"]:
+        return types.DecimalType()
+    if field_type in ["integer", "int"]:
+        return types.IntegerType()
+    if field_type == "long":
+        return types.LongType()
+    if field_type == "float":
+        return types.FloatType()
+    if field_type == "double":
+        return types.DoubleType()
+    if field_type == "boolean":
+        return types.BooleanType()
+    if field_type in ["timestamp", "timestamp_tz"]:
+        return types.TimestampType()
+    if field_type == "timestamp_ntz":
+        return types.TimestampNTZType()
+    if field_type == "date":
+        return types.DateType()
+    if field_type == "bytes":
+        return types.BinaryType()
+    return types.BinaryType()
+
+
+def print_schema(dtype: types.DataType) -> str:
+    """
+    Converts a PySpark DataType schema to its equivalent code representation.
+
+    Args:
+        dtype (types.DataType): The PySpark DataType schema to be converted.
+
+    Returns:
+        str: The code representation of the PySpark DataType schema.
+    """
+
+    def indent(text: str, level: int) -> str:
+        """
+        Indents each line of the given text by a specified number of levels.
+
+        Args:
+            text (str): The text to be indented.
+            level (int): The number of indentation levels.
+
+        Returns:
+            str: The indented text.
+        """
+        return "\n".join([f'{"    " * level}{line}' for line in text.split("\n")])
+
+    def repr_column(column: types.StructField) -> str:
+        """
+        Converts a PySpark StructField to its code representation.
+
+        Args:
+            column (types.StructField): The StructField to be converted.
+
+        Returns:
+            str: The code representation of the StructField.
+        """
+        name = f'"{column.name}"'
+        data_type = indent(print_schema(column.dataType), 1)
+        nullable = indent(f"{column.nullable}", 1)
+        return f"StructField({name},\n{data_type},\n{nullable}\n)"
+
+    def format_struct_type(struct_type: types.StructType) -> str:
+        """
+        Converts a PySpark StructType to its code representation.
+
+        Args:
+            struct_type (types.StructType): The StructType to be converted.
+
+        Returns:
+            str: The code representation of the StructType.
+        """
+        fields = ",\n".join([indent(repr_column(field), 1) for field in struct_type.fields])
+        return f"StructType([\n{fields}\n])"
+
+    if isinstance(dtype, types.StructType):
+        return format_struct_type(dtype)
+    elif isinstance(dtype, types.ArrayType):
+        return f"ArrayType({print_schema(dtype.elementType)})"
+    elif isinstance(dtype, types.DecimalType):
+        return f"DecimalType({dtype.precision}, {dtype.scale})"
+    else:
+        dtype_str = str(dtype)
+        return dtype_str if dtype_str.endswith("()") else f"{dtype_str}()"

datacontract/export/sql_converter.py

@@ -1,6 +1,29 @@
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification, Model
 
+from datacontract.export.exporter import Exporter, _check_models_for_export, _determine_sql_server_type
+
+
+class SqlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        server_type = _determine_sql_server_type(
+            data_contract,
+            sql_server_type,
+        )
+        return to_sql_ddl(data_contract, server_type, export_args.get("server"))
+
+
+class SqlQueryExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        server_type = _determine_sql_server_type(data_contract, sql_server_type, export_args.get("server"))
+        return to_sql_query(
+            data_contract,
+            model_name,
+            model_value,
+            server_type,
+        )
+
 
 def to_sql_query(
     data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake"
@@ -37,7 +60,9 @@ def _to_sql_query(model_name, model_value, server_type) -> str:
     return result
 
 
-def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str = "snowflake") -> str:
+def to_sql_ddl(
+    data_contract_spec: DataContractSpecification, server_type: str = "snowflake", server: str = None
+) -> str:
     if data_contract_spec is None:
         return ""
     if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
@@ -45,7 +70,12 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str =
 
     table_prefix = ""
 
-    for server_name, server in iter(data_contract_spec.servers.items()):
+    if server is None:
+        servers = data_contract_spec.servers
+    else:
+        servers = {server: data_contract_spec.servers[server]}
+
+    for server_name, server in iter(servers.items()):
         if server.type == "snowflake":
             server_type = "snowflake"
             break

datacontract/export/sql_type_converter.py

@@ -15,13 +15,15 @@ def convert_to_sql_type(field: Field, server_type: str) -> str:
         return convert_type_to_sqlserver(field)
     elif server_type == "bigquery":
         return convert_type_to_bigquery(field)
+    elif server_type == "trino":
+        return convert_type_to_trino(field)
     return field.type
 
 
 # snowflake data types:
 # https://docs.snowflake.com/en/sql-reference/data-types.html
 def convert_to_snowflake(field: Field) -> None | str:
-    if field.config and field.config["snowflakeType"] is not None:
+    if field.config and "snowflakeType" in field.config:
         return field.config["snowflakeType"]
 
     type = field.type
@@ -64,7 +66,7 @@ def convert_to_snowflake(field: Field) -> None | str:
 # https://www.postgresql.org/docs/current/datatype.html
 # Using the name whenever possible
 def convert_type_to_postgres(field: Field) -> None | str:
-    if field.config and field.config["postgresType"] is not None:
+    if field.config and "postgresType" in field.config:
         return field.config["postgresType"]
 
     type = field.type
@@ -109,7 +111,7 @@ def convert_type_to_postgres(field: Field) -> None | str:
 # databricks data types:
 # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
 def convert_to_databricks(field: Field) -> None | str:
-    if field.config and field.config["databricksType"] is not None:
+    if field.config and "databricksType" in field.config:
         return field.config["databricksType"]
     type = field.type
     if type is None:
@@ -161,8 +163,7 @@ def convert_to_duckdb(field: Field) -> None | str:
     if type.lower() in ["time"]:
         return "TIME"  # TIME WITHOUT TIME ZONE
     if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "DECIMAL"
+        return f"DECIMAL({field.precision},{field.scale})"
     if type.lower() in ["float"]:
         return "FLOAT"
     if type.lower() in ["double"]:
@@ -250,3 +251,29 @@ def get_type_config(field: Field, config_attr: str) -> dict[str, str] | None:
     if not field.config:
         return None
     return field.config.get(config_attr, None)
+
+
+def convert_type_to_trino(field: Field) -> None | str:
+    """Convert from supported datacontract types to equivalent trino types"""
+    field_type = field.type
+
+    if field_type.lower() in ["string", "text", "varchar"]:
+        return "varchar"
+    # tinyint, smallint not supported by data contract
+    if field_type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "decimal"
+    if field_type.lower() in ["int", "integer"]:
+        return "integer"
+    if field_type.lower() in ["long", "bigint"]:
+        return "bigint"
+    if field_type.lower() in ["float"]:
+        return "real"
+    if field_type.lower() in ["timestamp", "timestamp_tz"]:
+        return "timestamp(3) with time zone"
+    if field_type.lower() in ["timestamp_ntz"]:
+        return "timestamp(3)"
+    if field_type.lower() in ["bytes"]:
+        return "varbinary"
+    if field_type.lower() in ["object", "record", "struct"]:
+        return "json"

datacontract/export/terraform_converter.py

@@ -1,6 +1,12 @@
 import re
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
+from datacontract.export.exporter import Exporter
+
+
+class TerraformExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_terraform(data_contract)
 
 
 def to_terraform(data_contract_spec: DataContractSpecification, server_id: str = None) -> str:

datacontract/imports/avro_importer.py

@@ -1,9 +1,17 @@
 import avro.schema
 
+from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 from datacontract.model.exceptions import DataContractException
 
 
+class AvroImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        return import_avro(data_contract_specification, source)
+
+
 def import_avro(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
     if data_contract_specification.models is None:
         data_contract_specification.models = {}

datacontract/imports/bigquery_importer.py

@@ -1,12 +1,28 @@
 import json
+import logging
 from typing import List
 
-from google.cloud import bigquery
-
+from datacontract.imports.importer import Importer
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 from datacontract.model.exceptions import DataContractException
 
 
+class BigQueryImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        if source is not None:
+            data_contract_specification = import_bigquery_from_json(data_contract_specification, source)
+        else:
+            data_contract_specification = import_bigquery_from_api(
+                data_contract_specification,
+                import_args.get("bigquery_tables"),
+                import_args.get("bigquery_project"),
+                import_args.get("bigquery_dataset"),
+            )
+        return data_contract_specification
+
+
 def import_bigquery_from_json(
     data_contract_specification: DataContractSpecification, source: str
 ) -> DataContractSpecification:
@@ -30,6 +46,18 @@ def import_bigquery_from_api(
     bigquery_project: str,
     bigquery_dataset: str,
 ) -> DataContractSpecification:
+    try:
+        from google.cloud import bigquery
+    except ImportError as e:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="bigquery extra missing",
+            reason="Install the extra datacontract-cli[bigquery] to use bigquery",
+            engine="datacontract",
+            original_exception=e,
+        )
+
     client = bigquery.Client(project=bigquery_project)
 
     if bigquery_tables is None:
@@ -63,7 +91,7 @@ def import_bigquery_from_api(
     return data_contract_specification
 
 
-def fetch_table_names(client: bigquery.Client, dataset: str) -> List[str]:
+def fetch_table_names(client, dataset: str) -> List[str]:
     table_names = []
     api_tables = client.list_tables(dataset)
     for api_table in api_tables:
@@ -84,7 +112,9 @@ def convert_bigquery_schema(
     # what exactly leads to friendlyName being set
     table_id = bigquery_schema.get("tableReference").get("tableId")
 
-    data_contract_specification.models[table_id] = Model(fields=fields, type="table")
+    data_contract_specification.models[table_id] = Model(
+        fields=fields, type=map_bigquery_type(bigquery_schema.get("type"))
+    )
 
     # Copy the description, if it exists
     if bigquery_schema.get("description") is not None:
@@ -176,3 +206,16 @@ def map_type_from_bigquery(bigquery_type_str: str):
             reason=f"Unsupported type {bigquery_type_str} in bigquery json definition.",
             engine="datacontract",
         )
+
+
+def map_bigquery_type(bigquery_type: str) -> str:
+    if bigquery_type == "TABLE" or bigquery_type == "EXTERNAL" or bigquery_type == "SNAPSHOT":
+        return "table"
+    elif bigquery_type == "VIEW" or bigquery_type == "MATERIALIZED_VIEW":
+        return "view"
+    else:
+        logger = logging.getLogger(__name__)
+        logger.info(
+            f"Can't properly map bigquery table type '{bigquery_type}' to datacontracts model types. Mapping it to table."
+        )
+        return "table"