datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/export/sql_type_converter.py

@@ -1,19 +1,37 @@
+from datacontract.export.bigquery_converter import map_type_to_bigquery
 from datacontract.model.data_contract_specification import Field
 
 
 def convert_to_sql_type(field: Field, server_type: str) -> str:
+    if field.config and "physicalType" in field.config:
+        return field.config["physicalType"]
+
     if server_type == "snowflake":
         return convert_to_snowflake(field)
-    if server_type == "postgres":
+    elif server_type == "postgres":
         return convert_type_to_postgres(field)
-    if server_type == "databricks":
+    elif server_type == "dataframe":
+        return convert_to_dataframe(field)
+    elif server_type == "databricks":
         return convert_to_databricks(field)
+    elif server_type == "local" or server_type == "s3":
+        return convert_to_duckdb(field)
+    elif server_type == "sqlserver":
+        return convert_type_to_sqlserver(field)
+    elif server_type == "bigquery":
+        return convert_type_to_bigquery(field)
+    elif server_type == "trino":
+        return convert_type_to_trino(field)
+
     return field.type
 
 
 # snowflake data types:
 # https://docs.snowflake.com/en/sql-reference/data-types.html
-def convert_to_snowflake(field) -> None | str:
+def convert_to_snowflake(field: Field) -> None | str:
+    if field.config and "snowflakeType" in field.config:
+        return field.config["snowflakeType"]
+
     type = field.type
     # currently optimized for snowflake
     # LEARNING: data contract has no direct support for CHAR,CHARACTER
@@ -54,6 +72,9 @@ def convert_to_snowflake(field) -> None | str:
 # https://www.postgresql.org/docs/current/datatype.html
 # Using the name whenever possible
 def convert_type_to_postgres(field: Field) -> None | str:
+    if field.config and "postgresType" in field.config:
+        return field.config["postgresType"]
+
     type = field.type
     if type is None:
         return None
@@ -93,10 +114,62 @@ def convert_type_to_postgres(field: Field) -> None | str:
     return None
 
 
+# dataframe data types:
+# https://spark.apache.org/docs/latest/sql-ref-datatypes.html
+def convert_to_dataframe(field: Field) -> None | str:
+    if field.config and "dataframeType" in field.config:
+        return field.config["dataframeType"]
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "STRING"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP_NTZ"
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "STRING"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return f"DECIMAL({precision},{scale})"
+    if type.lower() in ["float"]:
+        return "FLOAT"
+    if type.lower() in ["double"]:
+        return "DOUBLE"
+    if type.lower() in ["integer", "int"]:
+        return "INT"
+    if type.lower() in ["long", "bigint"]:
+        return "BIGINT"
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_to_dataframe(nested_field)
+            nested_fields.append(f"{nested_field_name}:{nested_field_type}")
+        return f"STRUCT<{','.join(nested_fields)}>"
+    if type.lower() in ["bytes"]:
+        return "BINARY"
+    if type.lower() in ["array"]:
+        item_type = convert_to_dataframe(field.items)
+        return f"ARRAY<{item_type}>"
+    return None
+
+
 # databricks data types:
 # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
-def convert_to_databricks(field) -> None | str:
+def convert_to_databricks(field: Field) -> None | str:
     type = field.type
+    if (
+        field.config
+        and "databricksType" in field.config
+        and type.lower() not in ["array", "object", "record", "struct"]
+    ):
+        return field.config["databricksType"]
     if type is None:
         return None
     if type.lower() in ["string", "varchar", "text"]:
@@ -110,8 +183,9 @@ def convert_to_databricks(field) -> None | str:
     if type.lower() in ["time"]:
         return "STRING"
     if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "DECIMAL"
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return f"DECIMAL({precision},{scale})"
     if type.lower() in ["float"]:
         return "FLOAT"
     if type.lower() in ["double"]:
@@ -123,9 +197,196 @@ def convert_to_databricks(field) -> None | str:
     if type.lower() in ["boolean"]:
         return "BOOLEAN"
     if type.lower() in ["object", "record", "struct"]:
-        return "STRUCT"
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_to_databricks(nested_field)
+            nested_fields.append(f"{nested_field_name}:{nested_field_type}")
+        return f"STRUCT<{','.join(nested_fields)}>"
     if type.lower() in ["bytes"]:
         return "BINARY"
     if type.lower() in ["array"]:
-        return "ARRAY"
+        item_type = convert_to_databricks(field.items)
+        return f"ARRAY<{item_type}>"
+    if type.lower() in ["variant"]:
+        return "VARIANT"
     return None
+
+
+def convert_to_duckdb(field: Field) -> None | str:
+    """
+    Convert a data contract field to the corresponding DuckDB SQL type.
+
+    Parameters:
+    field (Field): The data contract field to convert.
+
+    Returns:
+    str: The corresponding DuckDB SQL type.
+    """
+    # Check
+    if field is None or field.type is None:
+        return None
+
+    # Get
+    type_lower = field.type.lower()
+
+    # Prepare
+    type_mapping = {
+        "varchar": "VARCHAR",
+        "string": "VARCHAR",
+        "text": "VARCHAR",
+        "binary": "BLOB",
+        "bytes": "BLOB",
+        "blob": "BLOB",
+        "boolean": "BOOLEAN",
+        "float": "FLOAT",
+        "double": "DOUBLE",
+        "int": "INTEGER",
+        "int32": "INTEGER",
+        "integer": "INTEGER",
+        "int64": "BIGINT",
+        "long": "BIGINT",
+        "bigint": "BIGINT",
+        "date": "DATE",
+        "time": "TIME",
+        "timestamp": "TIMESTAMP WITH TIME ZONE",
+        "timestamp_tz": "TIMESTAMP WITH TIME ZONE",
+        "timestamp_ntz": "TIMESTAMP",
+    }
+
+    # Convert simple mappings
+    if type_lower in type_mapping:
+        return type_mapping[type_lower]
+
+    # convert decimal numbers with precision and scale
+    if type_lower == "decimal" or type_lower == "number" or type_lower == "numeric":
+        return f"DECIMAL({field.precision},{field.scale})"
+
+    # Check list and map
+    if type_lower == "list" or type_lower == "array":
+        item_type = convert_to_duckdb(field.items)
+        return f"{item_type}[]"
+    if type_lower == "map":
+        key_type = convert_to_duckdb(field.keys)
+        value_type = convert_to_duckdb(field.values)
+        return f"MAP({key_type}, {value_type})"
+    if type_lower == "struct" or type_lower == "object" or type_lower == "record":
+        structure_field = "STRUCT("
+        field_strings = []
+        for fieldKey, fieldValue in field.fields.items():
+            field_strings.append(f"{fieldKey} {convert_to_duckdb(fieldValue)}")
+        structure_field += ", ".join(field_strings)
+        structure_field += ")"
+        return structure_field
+
+    # Return none
+    return None
+
+
+def convert_type_to_sqlserver(field: Field) -> None | str:
+    """Convert from supported datacontract types to equivalent sqlserver types"""
+    field_type = field.type
+    if not field_type:
+        return None
+
+    # If provided sql-server config type, prefer it over default mapping
+    if sqlserver_type := get_type_config(field, "sqlserverType"):
+        return sqlserver_type
+
+    field_type = field_type.lower()
+    if field_type in ["string", "varchar", "text"]:
+        if field.format == "uuid":
+            return "uniqueidentifier"
+        return "varchar"
+    if field_type in ["timestamp", "timestamp_tz"]:
+        return "datetimeoffset"
+    if field_type in ["timestamp_ntz"]:
+        if field.format == "datetime":
+            return "datetime"
+        return "datetime2"
+    if field_type in ["date"]:
+        return "date"
+    if field_type in ["time"]:
+        return "time"
+    if field_type in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        if field_type == "number":
+            return "numeric"
+        return field_type
+    if field_type in ["float"]:
+        return "float"
+    if field_type in ["double"]:
+        return "double precision"
+    if field_type in ["integer", "int", "bigint"]:
+        return field_type
+    if field_type in ["long"]:
+        return "bigint"
+    if field_type in ["boolean"]:
+        return "bit"
+    if field_type in ["object", "record", "struct"]:
+        return "jsonb"
+    if field_type in ["bytes"]:
+        return "binary"
+    if field_type in ["array"]:
+        raise NotImplementedError("SQLServer does not support array types.")
+    return None
+
+
+def convert_type_to_bigquery(field: Field) -> None | str:
+    """Convert from supported datacontract types to equivalent bigquery types"""
+
+    # BigQuery exporter cannot be used for complex types, as the exporter has different syntax than SodaCL
+
+    field_type = field.type
+    if not field_type:
+        return None
+
+    if field.config and "bigqueryType" in field.config:
+        return field.config["bigqueryType"]
+
+    if field_type.lower() in ["array"]:
+        item_type = convert_type_to_bigquery(field.items)
+        return f"ARRAY<{item_type}>"
+
+    if field_type.lower() in ["object", "record", "struct"]:
+        nested_fields = []
+        for nested_field_name, nested_field in field.fields.items():
+            nested_field_type = convert_type_to_bigquery(nested_field)
+            nested_fields.append(f"{nested_field_name} {nested_field_type}")
+        return f"STRUCT<{', '.join(nested_fields)}>"
+
+    return map_type_to_bigquery(field)
+
+
+def get_type_config(field: Field, config_attr: str) -> dict[str, str] | None:
+    """Retrieve type configuration if provided in datacontract."""
+    if not field.config:
+        return None
+    return field.config.get(config_attr, None)
+
+
+def convert_type_to_trino(field: Field) -> None | str:
+    """Convert from supported datacontract types to equivalent trino types"""
+    if field.config and "trinoType" in field.config:
+        return field.config["trinoType"]
+
+    field_type = field.type.lower()
+    if field_type in ["string", "text", "varchar"]:
+        return "varchar"
+    # tinyint, smallint not supported by data contract
+    if field_type in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "decimal"
+    if field_type in ["int", "integer"]:
+        return "integer"
+    if field_type in ["long", "bigint"]:
+        return "bigint"
+    if field_type in ["float"]:
+        return "real"
+    if field_type in ["timestamp", "timestamp_tz"]:
+        return "timestamp(3) with time zone"
+    if field_type in ["timestamp_ntz"]:
+        return "timestamp(3)"
+    if field_type in ["bytes"]:
+        return "varbinary"
+    if field_type in ["object", "record", "struct"]:
+        return "json"

datacontract/export/sqlalchemy_converter.py

@@ -0,0 +1,170 @@
+import ast
+import typing
+
+import datacontract.model.data_contract_specification as spec
+from datacontract.export.exporter import Exporter, _determine_sql_server_type
+
+
+class SQLAlchemyExporter(Exporter):
+    def export(
+        self, data_contract: spec.DataContractSpecification, model, server, sql_server_type, export_args
+    ) -> dict:
+        sql_server_type = _determine_sql_server_type(data_contract, sql_server_type, server)
+        return to_sqlalchemy_model_str(data_contract, sql_server_type, server)
+
+
+DECLARATIVE_BASE = "Base"
+
+
+def to_sqlalchemy_model_str(contract: spec.DataContractSpecification, sql_server_type: str = "", server=None) -> str:
+    server_obj = contract.servers.get(server)
+    classdefs = [
+        generate_model_class(model_name, model, server_obj, sql_server_type)
+        for (model_name, model) in contract.models.items()
+    ]
+    documentation = (
+        [ast.Expr(ast.Constant(contract.info.description))] if (contract.info and contract.info.description) else []
+    )
+
+    declarative_base = ast.ClassDef(
+        name=DECLARATIVE_BASE,
+        bases=[ast.Name(id="DeclarativeBase", ctx=ast.Load())],
+        body=[ast.Pass()],
+        keywords=[],
+        decorator_list=[],
+    )
+
+    databricks_timestamp = ast.ImportFrom(
+        module="databricks.sqlalchemy", names=[ast.alias("TIMESTAMP"), ast.alias("TIMESTAMP_NTZ")]
+    )
+    timestamp = ast.ImportFrom(module="sqlalchemy", names=[ast.alias(name="TIMESTAMP")])
+    result = ast.Module(
+        body=[
+            ast.ImportFrom(module="sqlalchemy.orm", names=[ast.alias(name="DeclarativeBase")]),
+            ast.ImportFrom(
+                module="sqlalchemy",
+                names=[
+                    ast.alias("Column"),
+                    ast.alias("Date"),
+                    ast.alias("Integer"),
+                    ast.alias("Numeric"),
+                    ast.alias("String"),
+                    ast.alias("Text"),
+                    ast.alias("VARCHAR"),
+                    ast.alias("BigInteger"),
+                    ast.alias("Float"),
+                    ast.alias("Double"),
+                    ast.alias("Boolean"),
+                    ast.alias("Date"),
+                    ast.alias("ARRAY"),
+                    ast.alias("LargeBinary"),
+                ],
+            ),
+            databricks_timestamp if sql_server_type == "databricks" else timestamp,
+            *documentation,
+            declarative_base,
+            *classdefs,
+        ],
+        type_ignores=[],
+    )
+    return ast.unparse(result)
+
+
+def Call(name, *args, **kwargs) -> ast.Call:
+    return ast.Call(
+        ast.Name(name),
+        args=[v for v in args],
+        keywords=[ast.keyword(arg=f"{k}", value=ast.Constant(v)) for (k, v) in kwargs.items()],
+    )
+
+
+def Column(predicate, **kwargs) -> ast.Call:
+    return Call("Column", predicate, **kwargs)
+
+
+def sqlalchemy_primitive(field: spec.Field):
+    sqlalchemy_name = {
+        "string": Call("String", ast.Constant(field.maxLength)),
+        "text": Call("Text", ast.Constant(field.maxLength)),
+        "varchar": Call("VARCHAR", ast.Constant(field.maxLength)),
+        "number": Call("Numeric", ast.Constant(field.precision), ast.Constant(field.scale)),
+        "decimal": Call("Numeric", ast.Constant(field.precision), ast.Constant(field.scale)),
+        "numeric": Call("Numeric", ast.Constant(field.precision), ast.Constant(field.scale)),
+        "int": ast.Name("Integer"),
+        "integer": ast.Name("Integer"),
+        "long": ast.Name("BigInteger"),
+        "bigint": ast.Name("BigInteger"),
+        "float": ast.Name("Float"),
+        "double": ast.Name("Double"),
+        "boolean": ast.Name("Boolean"),
+        "timestamp": ast.Name("TIMESTAMP"),
+        "timestamp_tz": Call("TIMESTAMP", ast.Constant(True)),
+        "timestamp_ntz": ast.Name("TIMESTAMP_NTZ"),
+        "date": ast.Name("Date"),
+        "bytes": Call("LargeBinary", ast.Constant(field.maxLength)),
+    }
+    return sqlalchemy_name.get(field.type)
+
+
+def constant_field_value(field_name: str, field: spec.Field) -> tuple[ast.Call, typing.Optional[ast.ClassDef]]:
+    new_type = sqlalchemy_primitive(field)
+    match field.type:
+        case "array":
+            new_type = Call("ARRAY", sqlalchemy_primitive(field.items))
+    if new_type is None:
+        raise RuntimeError(f"Unsupported field type {field.type}.")
+
+    return Column(
+        new_type, nullable=not field.required, comment=field.description, primary_key=field.primaryKey or field.primary
+    ), None
+
+
+def column_assignment(field_name: str, field: spec.Field) -> tuple[ast.Call, typing.Optional[ast.ClassDef]]:
+    return constant_field_value(field_name, field)
+
+
+def is_simple_field(field: spec.Field) -> bool:
+    return field.type not in set(["object", "record", "struct"])
+
+
+def field_definitions(fields: dict[str, spec.Field]) -> tuple[list[ast.Expr], list[ast.ClassDef]]:
+    annotations: list[ast.Expr] = []
+    classes: list[typing.Any] = []
+    for field_name, field in fields.items():
+        (ann, new_class) = column_assignment(field_name, field)
+        annotations.append(ast.Assign(targets=[ast.Name(id=field_name, ctx=ast.Store())], value=ann, lineno=0))
+    return (annotations, classes)
+
+
+def generate_model_class(
+    name: str, model_definition: spec.Model, server=None, sql_server_type: str = ""
+) -> ast.ClassDef:
+    (field_assignments, nested_classes) = field_definitions(model_definition.fields)
+    documentation = [ast.Expr(ast.Constant(model_definition.description))] if model_definition.description else []
+
+    schema = None if server is None else server.schema_
+    table_name = ast.Constant(name)
+    if sql_server_type == "databricks":
+        table_name = ast.Constant(name.lower())
+
+    result = ast.ClassDef(
+        name=name.capitalize(),
+        bases=[ast.Name(id=DECLARATIVE_BASE, ctx=ast.Load())],
+        body=[
+            *documentation,
+            ast.Assign(targets=[ast.Name("__tablename__")], value=table_name, lineno=0),
+            ast.Assign(
+                targets=[ast.Name("__table_args__")],
+                value=ast.Dict(
+                    keys=[ast.Constant("comment"), ast.Constant("schema")],
+                    values=[ast.Constant(model_definition.description), ast.Constant(schema)],
+                ),
+                lineno=0,
+            ),
+            *nested_classes,
+            *field_assignments,
+        ],
+        keywords=[],
+        decorator_list=[],
+    )
+    return result

datacontract/export/terraform_converter.py

@@ -1,7 +1,12 @@
 import re
 
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Server
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification, Server
+
+
+class TerraformExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_terraform(data_contract)
 
 
 def to_terraform(data_contract_spec: DataContractSpecification, server_id: str = None) -> str: