datacontract-cli 0.10.11__py3-none-any.whl → 0.10.13__py3-none-any.whl

datacontract/export/exporter_factory.py

@@ -62,6 +62,12 @@ exporter_factory.register_lazy_exporter(
     class_name="BigQueryExporter",
 )
 
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.data_caterer,
+    module_path="datacontract.export.data_caterer_converter",
+    class_name="DataCatererExporter",
+)
+
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.dbml, module_path="datacontract.export.dbml_converter", class_name="DbmlExporter"
 )
@@ -143,3 +149,9 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.spark, module_path="datacontract.export.spark_converter", class_name="SparkExporter"
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sqlalchemy,
+    module_path="datacontract.export.sqlalchemy_converter",
+    class_name="SQLAlchemyExporter",
+)

datacontract/export/jsonschema_converter.py

@@ -36,7 +36,19 @@ def to_property(field: Field) -> dict:
     property = {}
     json_type, json_format = convert_type_format(field.type, field.format)
     if json_type is not None:
-        property["type"] = json_type
+        if not field.required:
+            """
+            From: https://json-schema.org/understanding-json-schema/reference/type
+            The type keyword may either be a string or an array:
+
+            If it's a string, it is the name of one of the basic types above.
+            If it is an array, it must be an array of strings, where each string 
+            is the name of one of the basic types, and each element is unique.
+            In this case, the JSON snippet is valid if it matches any of the given types.
+            """
+            property["type"] = [json_type, "null"]
+        else:
+            property["type"] = json_type
     if json_format is not None:
         property["format"] = json_format
     if field.unique:
@@ -50,7 +62,6 @@ def to_property(field: Field) -> dict:
         property["required"] = to_required(field.fields)
     if json_type == "array":
         property["items"] = to_property(field.items)
-
     if field.pattern:
         property["pattern"] = field.pattern
     if field.enum:

datacontract/export/spark_converter.py

@@ -123,10 +123,12 @@ def to_data_type(field: Field) -> types.DataType:
         return types.ArrayType(to_data_type(field.items))
     if field_type in ["object", "record", "struct"]:
         return types.StructType(to_struct_type(field.fields))
+    if field_type == "map":
+        return types.MapType(to_data_type(field.keys), to_data_type(field.values))
     if field_type in ["string", "varchar", "text"]:
         return types.StringType()
     if field_type in ["number", "decimal", "numeric"]:
-        return types.DecimalType()
+        return types.DecimalType(precision=field.precision, scale=field.scale)
     if field_type in ["integer", "int"]:
         return types.IntegerType()
     if field_type == "long":
@@ -204,6 +206,8 @@ def print_schema(dtype: types.DataType) -> str:
         return format_struct_type(dtype)
     elif isinstance(dtype, types.ArrayType):
         return f"ArrayType({print_schema(dtype.elementType)})"
+    elif isinstance(dtype, types.MapType):
+        return f"MapType(\n{indent(print_schema(dtype.keyType), 1)}, {print_schema(dtype.valueType)})"
     elif isinstance(dtype, types.DecimalType):
         return f"DecimalType({dtype.precision}, {dtype.scale})"
     else:

datacontract/export/sql_type_converter.py

@@ -149,37 +149,72 @@ def convert_to_databricks(field: Field) -> None | str:
 
 
 def convert_to_duckdb(field: Field) -> None | str:
-    type = field.type
-    if type is None:
+    """
+    Convert a data contract field to the corresponding DuckDB SQL type.
+
+    Parameters:
+    field (Field): The data contract field to convert.
+
+    Returns:
+    str: The corresponding DuckDB SQL type.
+    """
+    # Check
+    if field is None or field.type is None:
         return None
-    if type.lower() in ["string", "varchar", "text"]:
-        return "VARCHAR"  # aliases: VARCHAR, CHAR, BPCHAR, STRING, TEXT, VARCHAR(n)	STRING(n), TEXT(n)
-    if type.lower() in ["timestamp", "timestamp_tz"]:
-        return "TIMESTAMP WITH TIME ZONE"  # aliases: TIMESTAMPTZ
-    if type.lower() in ["timestamp_ntz"]:
-        return "DATETIME"  # timestamp with microsecond precision (ignores time zone), aliases: TIMESTAMP
-    if type.lower() in ["date"]:
-        return "DATE"
-    if type.lower() in ["time"]:
-        return "TIME"  # TIME WITHOUT TIME ZONE
-    if type.lower() in ["number", "decimal", "numeric"]:
+
+    # Get
+    type_lower = field.type.lower()
+
+    # Prepare
+    type_mapping = {
+        "varchar": "VARCHAR",
+        "string": "VARCHAR",
+        "text": "VARCHAR",
+        "binary": "BLOB",
+        "bytes": "BLOB",
+        "blob": "BLOB",
+        "boolean": "BOOLEAN",
+        "float": "FLOAT",
+        "double": "DOUBLE",
+        "int": "INTEGER",
+        "int32": "INTEGER",
+        "integer": "INTEGER",
+        "int64": "BIGINT",
+        "long": "BIGINT",
+        "bigint": "BIGINT",
+        "date": "DATE",
+        "time": "TIME",
+        "timestamp": "TIMESTAMP WITH TIME ZONE",
+        "timestamp_tz": "TIMESTAMP WITH TIME ZONE",
+        "timestamp_ntz": "DATETIME",
+    }
+
+    # Convert simple mappings
+    if type_lower in type_mapping:
+        return type_mapping[type_lower]
+
+    # convert decimal numbers with precision and scale
+    if type_lower == "decimal" or type_lower == "number" or type_lower == "numeric":
         return f"DECIMAL({field.precision},{field.scale})"
-    if type.lower() in ["float"]:
-        return "FLOAT"
-    if type.lower() in ["double"]:
-        return "DOUBLE"
-    if type.lower() in ["integer", "int"]:
-        return "INT"
-    if type.lower() in ["long", "bigint"]:
-        return "BIGINT"
-    if type.lower() in ["boolean"]:
-        return "BOOLEAN"
-    if type.lower() in ["object", "record", "struct"]:
-        return "STRUCT"
-    if type.lower() in ["bytes"]:
-        return "BLOB"
-    if type.lower() in ["array"]:
-        return "ARRAY"
+
+    # Check list and map
+    if type_lower == "list" or type_lower == "array":
+        item_type = convert_to_duckdb(field.items)
+        return f"{item_type}[]"
+    if type_lower == "map":
+        key_type = convert_to_duckdb(field.keys)
+        value_type = convert_to_duckdb(field.values)
+        return f"MAP({key_type}, {value_type})"
+    if type_lower == "struct" or type_lower == "object" or type_lower == "record":
+        structure_field = "STRUCT("
+        field_strings = []
+        for fieldKey, fieldValue in field.fields.items():
+            field_strings.append(f"{fieldKey} {convert_to_duckdb(fieldValue)}")
+        structure_field += ", ".join(field_strings)
+        structure_field += ")"
+        return structure_field
+
+    # Return none
     return None
 
 
@@ -234,16 +269,7 @@ def convert_type_to_sqlserver(field: Field) -> None | str:
 
 def convert_type_to_bigquery(field: Field) -> None | str:
     """Convert from supported datacontract types to equivalent bigquery types"""
-    field_type = field.type
-    if not field_type:
-        return None
-
-    # If provided sql-server config type, prefer it over default mapping
-    if bigquery_type := get_type_config(field, "bigqueryType"):
-        return bigquery_type
-
-    field_type = field_type.lower()
-    return map_type_to_bigquery(field_type, field.title)
+    return map_type_to_bigquery(field)
 
 
 def get_type_config(field: Field, config_attr: str) -> dict[str, str] | None:

datacontract/export/sqlalchemy_converter.py

@@ -0,0 +1,169 @@
+import ast
+import typing
+
+import datacontract.model.data_contract_specification as spec
+from datacontract.export.exporter import Exporter
+from datacontract.export.exporter import _determine_sql_server_type
+
+
+class SQLAlchemyExporter(Exporter):
+    def export(
+        self, data_contract: spec.DataContractSpecification, model, server, sql_server_type, export_args
+    ) -> dict:
+        sql_server_type = _determine_sql_server_type(data_contract, sql_server_type, server)
+        return to_sqlalchemy_model_str(data_contract, sql_server_type, server)
+
+
+DECLARATIVE_BASE = "Base"
+
+
+def to_sqlalchemy_model_str(contract: spec.DataContractSpecification, sql_server_type: str = "", server=None) -> str:
+    server_obj = contract.servers.get(server)
+    classdefs = [
+        generate_model_class(model_name, model, server_obj, sql_server_type)
+        for (model_name, model) in contract.models.items()
+    ]
+    documentation = (
+        [ast.Expr(ast.Constant(contract.info.description))] if (contract.info and contract.info.description) else []
+    )
+
+    declarative_base = ast.ClassDef(
+        name=DECLARATIVE_BASE,
+        bases=[ast.Name(id="DeclarativeBase", ctx=ast.Load())],
+        body=[ast.Pass()],
+        keywords=[],
+        decorator_list=[],
+    )
+
+    databricks_timestamp = ast.ImportFrom(
+        module="databricks.sqlalchemy", names=[ast.alias("TIMESTAMP"), ast.alias("TIMESTAMP_NTZ")]
+    )
+    timestamp = ast.ImportFrom(module="sqlalchemy", names=[ast.alias(name="TIMESTAMP")])
+    result = ast.Module(
+        body=[
+            ast.ImportFrom(module="sqlalchemy.orm", names=[ast.alias(name="DeclarativeBase")]),
+            ast.ImportFrom(
+                module="sqlalchemy",
+                names=[
+                    ast.alias("Column"),
+                    ast.alias("Date"),
+                    ast.alias("Integer"),
+                    ast.alias("Numeric"),
+                    ast.alias("String"),
+                    ast.alias("Text"),
+                    ast.alias("VARCHAR"),
+                    ast.alias("BigInteger"),
+                    ast.alias("Float"),
+                    ast.alias("Double"),
+                    ast.alias("Boolean"),
+                    ast.alias("Date"),
+                    ast.alias("ARRAY"),
+                    ast.alias("LargeBinary"),
+                ],
+            ),
+            databricks_timestamp if sql_server_type == "databricks" else timestamp,
+            *documentation,
+            declarative_base,
+            *classdefs,
+        ],
+        type_ignores=[],
+    )
+    return ast.unparse(result)
+
+
+def Call(name, *args, **kwargs) -> ast.Call:
+    return ast.Call(
+        ast.Name(name),
+        args=[v for v in args],
+        keywords=[ast.keyword(arg=f"{k}", value=ast.Constant(v)) for (k, v) in kwargs.items()],
+    )
+
+
+def Column(predicate, **kwargs) -> ast.Call:
+    return Call("Column", predicate, **kwargs)
+
+
+def sqlalchemy_primitive(field: spec.Field):
+    sqlalchemy_name = {
+        "string": Call("String", ast.Constant(field.maxLength)),
+        "text": Call("Text", ast.Constant(field.maxLength)),
+        "varchar": Call("VARCHAR", ast.Constant(field.maxLength)),
+        "number": Call("Numeric", ast.Constant(field.precision), ast.Constant(field.scale)),
+        "decimal": Call("Numeric", ast.Constant(field.precision), ast.Constant(field.scale)),
+        "numeric": Call("Numeric", ast.Constant(field.precision), ast.Constant(field.scale)),
+        "int": ast.Name("Integer"),
+        "integer": ast.Name("Integer"),
+        "long": ast.Name("BigInteger"),
+        "bigint": ast.Name("BigInteger"),
+        "float": ast.Name("Float"),
+        "double": ast.Name("Double"),
+        "boolean": ast.Name("Boolean"),
+        "timestamp": ast.Name("TIMESTAMP"),
+        "timestamp_tz": Call("TIMESTAMP", ast.Constant(True)),
+        "timestamp_ntz": ast.Name("TIMESTAMP_NTZ"),
+        "date": ast.Name("Date"),
+        "bytes": Call("LargeBinary", ast.Constant(field.maxLength)),
+    }
+    return sqlalchemy_name.get(field.type)
+
+
+def constant_field_value(field_name: str, field: spec.Field) -> tuple[ast.Call, typing.Optional[ast.ClassDef]]:
+    new_type = sqlalchemy_primitive(field)
+    match field.type:
+        case "array":
+            new_type = Call("ARRAY", sqlalchemy_primitive(field.items))
+    if new_type is None:
+        raise RuntimeError(f"Unsupported field type {field.type}.")
+
+    return Column(new_type, nullable=not field.required, comment=field.description, primary_key=field.primary), None
+
+
+def column_assignment(field_name: str, field: spec.Field) -> tuple[ast.Call, typing.Optional[ast.ClassDef]]:
+    return constant_field_value(field_name, field)
+
+
+def is_simple_field(field: spec.Field) -> bool:
+    return field.type not in set(["object", "record", "struct"])
+
+
+def field_definitions(fields: dict[str, spec.Field]) -> tuple[list[ast.Expr], list[ast.ClassDef]]:
+    annotations: list[ast.Expr] = []
+    classes: list[typing.Any] = []
+    for field_name, field in fields.items():
+        (ann, new_class) = column_assignment(field_name, field)
+        annotations.append(ast.Assign(targets=[ast.Name(id=field_name, ctx=ast.Store())], value=ann, lineno=0))
+    return (annotations, classes)
+
+
+def generate_model_class(
+    name: str, model_definition: spec.Model, server=None, sql_server_type: str = ""
+) -> ast.ClassDef:
+    (field_assignments, nested_classes) = field_definitions(model_definition.fields)
+    documentation = [ast.Expr(ast.Constant(model_definition.description))] if model_definition.description else []
+
+    schema = None if server is None else server.schema_
+    table_name = ast.Constant(name)
+    if sql_server_type == "databricks":
+        table_name = ast.Constant(name.lower())
+
+    result = ast.ClassDef(
+        name=name.capitalize(),
+        bases=[ast.Name(id=DECLARATIVE_BASE, ctx=ast.Load())],
+        body=[
+            *documentation,
+            ast.Assign(targets=[ast.Name("__tablename__")], value=table_name, lineno=0),
+            ast.Assign(
+                targets=[ast.Name("__table_args__")],
+                value=ast.Dict(
+                    keys=[ast.Constant("comment"), ast.Constant("schema")],
+                    values=[ast.Constant(model_definition.description), ast.Constant(schema)],
+                ),
+                lineno=0,
+            ),
+            *nested_classes,
+            *field_assignments,
+        ],
+        keywords=[],
+        decorator_list=[],
+    )
+    return result

datacontract/imports/avro_importer.py

@@ -133,6 +133,7 @@ def import_record_fields(record_fields: List[avro.schema.Field]) -> Dict[str, Fi
         elif field.type.type == "enum":
             imported_field.type = "string"
             imported_field.enum = field.type.symbols
+            imported_field.title = field.type.name
             if not imported_field.config:
                 imported_field.config = {}
             imported_field.config["avroType"] = "enum"

datacontract/imports/bigquery_importer.py

@@ -10,13 +10,13 @@ from datacontract.model.exceptions import DataContractException
 class BigQueryImporter(Importer):
     def import_source(
         self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
-    ) -> dict:
+    ) -> DataContractSpecification:
         if source is not None:
             data_contract_specification = import_bigquery_from_json(data_contract_specification, source)
         else:
             data_contract_specification = import_bigquery_from_api(
                 data_contract_specification,
-                import_args.get("bigquery_tables"),
+                import_args.get("bigquery_table"),
                 import_args.get("bigquery_project"),
                 import_args.get("bigquery_dataset"),
             )

datacontract/imports/dbml_importer.py

@@ -0,0 +1,112 @@
+from pydbml import PyDBML, Database
+from typing import List
+
+from pyparsing import ParseException
+
+from datacontract.imports.importer import Importer
+from datacontract.imports.sql_importer import map_type_from_sql
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.exceptions import DataContractException
+
+
+class DBMLImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> DataContractSpecification:
+        data_contract_specification = import_dbml_from_source(
+            data_contract_specification,
+            source,
+            import_args.get("dbml_schema"),
+            import_args.get("dbml_table"),
+        )
+        return data_contract_specification
+
+
+def import_dbml_from_source(
+    data_contract_specification: DataContractSpecification,
+    source: str,
+    import_schemas: List[str],
+    import_tables: List[str],
+) -> DataContractSpecification:
+    try:
+        with open(source, "r") as file:
+            dbml_schema = PyDBML(file)
+    except ParseException as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse DBML schema",
+            reason=f"Failed to parse DBML schema from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    return convert_dbml(data_contract_specification, dbml_schema, import_schemas, import_tables)
+
+
+def convert_dbml(
+    data_contract_specification: DataContractSpecification,
+    dbml_schema: Database,
+    import_schemas: List[str],
+    import_tables: List[str],
+) -> DataContractSpecification:
+    if dbml_schema.project is not None:
+        data_contract_specification.info.title = dbml_schema.project.name
+
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    for table in dbml_schema.tables:
+        schema_name = table.schema
+        table_name = table.name
+
+        # Skip if import schemas or table names are defined
+        # and the current table doesn't match
+        # if empty no filtering is done
+        if import_schemas and schema_name not in import_schemas:
+            continue
+
+        if import_tables and table_name not in import_tables:
+            continue
+
+        fields = import_table_fields(table, dbml_schema.refs)
+
+        data_contract_specification.models[table_name] = Model(
+            fields=fields, namespace=schema_name, description=table.note.text
+        )
+
+    return data_contract_specification
+
+
+def import_table_fields(table, references) -> dict[str, Field]:
+    imported_fields = {}
+    for field in table.columns:
+        field_name = field.name
+        imported_fields[field_name] = Field()
+        imported_fields[field_name].required = field.not_null
+        imported_fields[field_name].description = field.note.text
+        imported_fields[field_name].primary = field.pk
+        imported_fields[field_name].unique = field.unique
+        # This is an assumption, that these might be valid SQL Types, since
+        # DBML doesn't really enforce anything other than 'no spaces' in column types
+        imported_fields[field_name].type = map_type_from_sql(field.type)
+
+        ref = get_reference(field, references)
+        if ref is not None:
+            imported_fields[field_name].references = ref
+
+    return imported_fields
+
+
+def get_reference(field, references):
+    result = None
+    for ref in references:
+        ref_table_name = ref.col1[0].table.name
+        ref_col_name = ref.col1[0].name
+        field_table_name = field.table.name
+        field_name = field.name
+
+        if ref_table_name == field_table_name and ref_col_name == field_name:
+            result = f"{ref.col2[0].table.name}.{ref.col2[0].name}"
+            return result
+
+    return result