datacontract-cli 0.9.7__py3-none-any.whl → 0.9.9__py3-none-any.whl

datacontract/export/protobuf_converter.py

@@ -1,9 +1,4 @@
-from typing import Dict
-
-import yaml
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 def to_protobuf(data_contract_spec: DataContractSpecification):
@@ -24,7 +19,7 @@ def _to_protobuf_message_name(model_name):
     return model_name[0].upper() + model_name[1:]
 
 
-def to_protobuf_message(model_name, fields, description, indent_level:int = 0):
+def to_protobuf_message(model_name, fields, description, indent_level: int = 0):
     result = ""
 
     if description is not None:
@@ -34,8 +29,15 @@ def to_protobuf_message(model_name, fields, description, indent_level:int = 0):
     number = 1
     for field_name, field in fields.items():
         if field.type in ["object", "record", "struct"]:
-            fields_protobuf += "\n".join(
-                map(lambda x: "  " + x, to_protobuf_message(field_name, field.fields, field.description, indent_level + 1).splitlines())) + "\n"
+            fields_protobuf += (
+                "\n".join(
+                    map(
+                        lambda x: "  " + x,
+                        to_protobuf_message(field_name, field.fields, field.description, indent_level + 1).splitlines(),
+                    )
+                )
+                + "\n"
+            )
 
         fields_protobuf += to_protobuf_field(field_name, field, field.description, number, 1) + "\n"
         number += 1
@@ -44,7 +46,7 @@ def to_protobuf_message(model_name, fields, description, indent_level:int = 0):
     return result
 
 
-def to_protobuf_field(field_name, field, description, number:int, indent_level:int = 0):
+def to_protobuf_field(field_name, field, description, number: int, indent_level: int = 0):
     optional = ""
     if not field.required:
         optional = "optional "

datacontract/export/pydantic_converter.py

@@ -0,0 +1,131 @@
+import ast
+import typing
+
+import datacontract.model.data_contract_specification as spec
+
+
+def to_pydantic_model_str(contract: spec.DataContractSpecification) -> str:
+    classdefs = [generate_model_class(model_name, model) for (model_name, model) in contract.models.items()]
+    documentation = (
+        [ast.Expr(ast.Constant(contract.info.description))] if (contract.info and contract.info.description) else []
+    )
+    result = ast.Module(
+        body=[
+            ast.Import(
+                names=[
+                    ast.Name("datetime", ctx=ast.Load()),
+                    ast.Name("typing", ctx=ast.Load()),
+                    ast.Name("pydantic", ctx=ast.Load()),
+                ]
+            ),
+            *documentation,
+            *classdefs,
+        ],
+        type_ignores=[],
+    )
+    return ast.unparse(result)
+
+
+def optional_of(node) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Attribute(ast.Name(id="typing", ctx=ast.Load()), attr="Optional", ctx=ast.Load()), slice=node
+    )
+
+
+def list_of(node) -> ast.Subscript:
+    return ast.Subscript(value=ast.Name(id="list", ctx=ast.Load()), slice=node)
+
+
+def product_of(nodes: list[typing.Any]) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Attribute(value=ast.Name(id="typing", ctx=ast.Load()), attr="Product", ctx=ast.Load()),
+        slice=ast.Tuple(nodes, ctx=ast.Load()),
+    )
+
+
+type_annotation_type = typing.Union[ast.Name, ast.Attribute, ast.Constant, ast.Subscript]
+
+
+def constant_field_annotation(
+    field_name: str, field: spec.Field
+) -> tuple[type_annotation_type, typing.Optional[ast.ClassDef]]:
+    match field.type:
+        case "string" | "text" | "varchar":
+            return (ast.Name("str", ctx=ast.Load()), None)
+        case "number", "decimal", "numeric":
+            # Either integer or float in specification,
+            # so we use float.
+            return (ast.Name("float", ctx=ast.Load()), None)
+        case "int" | "integer" | "long" | "bigint":
+            return (ast.Name("int", ctx=ast.Load()), None)
+        case "float" | "double":
+            return (ast.Name("float", ctx=ast.Load()), None)
+        case "boolean":
+            return (ast.Name("bool", ctx=ast.Load()), None)
+        case "timestamp" | "timestamp_tz" | "timestamp_ntz":
+            return (ast.Attribute(value=ast.Name(id="datetime", ctx=ast.Load()), attr="datetime"), None)
+        case "date":
+            return (ast.Attribute(value=ast.Name(id="datetime", ctx=ast.Load()), attr="date"), None)
+        case "bytes":
+            return (ast.Name("bytes", ctx=ast.Load()), None)
+        case "null":
+            return (ast.Constant("None"), None)
+        case "array":
+            (annotated_type, new_class) = type_annotation(field_name, field.items)
+            return (list_of(annotated_type), new_class)
+        case "object" | "record" | "struct":
+            classdef = generate_field_class(field_name.capitalize(), field)
+            return (ast.Name(field_name.capitalize(), ctx=ast.Load()), classdef)
+        case _:
+            raise RuntimeError(f"Unsupported field type {field.type}.")
+
+
+def type_annotation(field_name: str, field: spec.Field) -> tuple[type_annotation_type, typing.Optional[ast.ClassDef]]:
+    if field.required:
+        return constant_field_annotation(field_name, field)
+    else:
+        (annotated_type, new_classes) = constant_field_annotation(field_name, field)
+        return (optional_of(annotated_type), new_classes)
+
+
+def is_simple_field(field: spec.Field) -> bool:
+    return field.type not in set(["object", "record", "struct"])
+
+
+def field_definitions(fields: dict[str, spec.Field]) -> tuple[list[ast.Expr], list[ast.ClassDef]]:
+    annotations = []
+    classes = []
+    for field_name, field in fields.items():
+        (ann, new_class) = type_annotation(field_name, field)
+        annotations.append(ast.AnnAssign(target=ast.Name(id=field_name, ctx=ast.Store()), annotation=ann, simple=1))
+        if field.description and is_simple_field(field):
+            annotations.append(ast.Expr(ast.Constant(field.description)))
+        if new_class:
+            classes.append(new_class)
+    return (annotations, classes)
+
+
+def generate_field_class(field_name: str, field: spec.Field) -> ast.ClassDef:
+    assert field.type in set(["object", "record", "struct"])
+    (annotated_type, new_classes) = field_definitions(field.fields)
+    documentation = [ast.Expr(ast.Constant(field.description))] if field.description else []
+    return ast.ClassDef(
+        name=field_name,
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()), attr="BaseModel", ctx=ast.Load())],
+        body=[*documentation, *new_classes, *annotated_type],
+        keywords=[],
+        decorator_list=[],
+    )
+
+
+def generate_model_class(name: str, model_definition: spec.Model) -> ast.ClassDef:
+    (field_assignments, nested_classes) = field_definitions(model_definition.fields)
+    documentation = [ast.Expr(ast.Constant(model_definition.description))] if model_definition.description else []
+    result = ast.ClassDef(
+        name=name.capitalize(),
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()), attr="BaseModel", ctx=ast.Load())],
+        body=[*documentation, *nested_classes, *field_assignments],
+        keywords=[],
+        decorator_list=[],
+    )
+    return result

datacontract/export/rdf_converter.py

@@ -1,17 +1,39 @@
-from typing import Dict
-import inspect
 from pydantic import BaseModel
 from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
 
 from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
+    DataContractSpecification
 
 
 def is_literal(property_name):
-    return property_name in ["dataContractSpecification", "title", "version", "description", "name", "url", "type",
-                             "location", "format", "delimiter", "usage", "limitations",
-                             "billing", "noticePeriod", "required", "unique", "minLength", "maxLength", "example",
-                             "pii", "classification", "data", "enum", "minimum", "maximum", "patterns"]
+    return property_name in [
+        "dataContractSpecification",
+        "title",
+        "version",
+        "description",
+        "name",
+        "url",
+        "type",
+        "location",
+        "format",
+        "delimiter",
+        "usage",
+        "limitations",
+        "billing",
+        "noticePeriod",
+        "required",
+        "unique",
+        "minLength",
+        "maxLength",
+        "example",
+        "pii",
+        "classification",
+        "data",
+        "enum",
+        "minimum",
+        "maximum",
+        "patterns",
+    ]
 
 
 def is_uriref(property_name):
@@ -21,6 +43,7 @@ def is_uriref(property_name):
 def to_rdf_n3(data_contract_spec: DataContractSpecification, base) -> str:
     return to_rdf(data_contract_spec, base).serialize(format="n3")
 
+
 def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
     if base is not None:
         g = Graph(base=base)
@@ -61,7 +84,7 @@ def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
 
 def add_example(contract, example, graph, dc, dcx):
     an_example = BNode()
-    graph.add((contract, dc['example'], an_example))
+    graph.add((contract, dc["example"], an_example))
     graph.add((an_example, RDF.type, URIRef(dc + "Example")))
     for example_property in example.model_fields:
         add_triple(sub=an_example, pred=example_property, obj=example, graph=graph, dc=dc, dcx=dcx)
@@ -81,14 +104,14 @@ def add_triple(sub, pred, obj, graph, dc, dcx):
 
 def add_model(contract, model, model_name, graph, dc, dcx):
     a_model = URIRef(model_name)
-    graph.add((contract, dc['model'], a_model))
+    graph.add((contract, dc["model"], a_model))
     graph.add((a_model, dc.description, Literal(model.description)))
     graph.add((a_model, RDF.type, URIRef(dc + "Model")))
     for field_name, field in model.fields.items():
         a_field = BNode()
-        graph.add((a_model, dc['field'], a_field))
+        graph.add((a_model, dc["field"], a_field))
         graph.add((a_field, RDF.type, URIRef(dc + "Field")))
-        graph.add((a_field, dc['name'], Literal(field_name)))
+        graph.add((a_field, dc["name"], Literal(field_name)))
         for field_property in field.model_fields:
             add_triple(sub=a_field, pred=field_property, obj=field, graph=graph, dc=dc, dcx=dcx)
 

datacontract/export/sodacl_converter.py

@@ -1,14 +1,17 @@
 import yaml
 
+from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import \
     DataContractSpecification
 
 
-def to_sodacl_yaml(data_contract_spec: DataContractSpecification, check_types: bool = True) -> str:
+def to_sodacl_yaml(
+    data_contract_spec: DataContractSpecification, server_type: str = None, check_types: bool = True
+) -> str:
     try:
         sodacl = {}
         for model_key, model_value in data_contract_spec.models.items():
-            k, v = to_checks(model_key, model_value, check_types)
+            k, v = to_checks(model_key, model_value, server_type, check_types)
             sodacl[k] = v
         add_quality_checks(sodacl, data_contract_spec)
         sodacl_yaml_str = yaml.dump(sodacl, default_flow_style=False, sort_keys=False)
@@ -17,17 +20,41 @@ def to_sodacl_yaml(data_contract_spec: DataContractSpecification, check_types: b
         return f"Error: {e}"
 
 
-def to_checks(model_key, model_value, check_types: bool):
+def to_checks(model_key, model_value, server_type: str, check_types: bool):
     checks = []
     fields = model_value.fields
+
+    quote_field_name = server_type in ["postgres"]
+
     for field_name, field in fields.items():
         checks.append(check_field_is_present(field_name))
         if check_types and field.type is not None:
-            checks.append(check_field_type(field_name, field.type))
+            sql_type = convert_to_sql_type(field, server_type)
+            checks.append(check_field_type(field_name, sql_type))
         if field.required:
-            checks.append(check_field_required(field_name))
+            checks.append(check_field_required(field_name, quote_field_name))
         if field.unique:
-            checks.append(check_field_unique(field_name))
+            checks.append(check_field_unique(field_name, quote_field_name))
+        if field.minLength is not None:
+            checks.append(check_field_min_length(field_name, field.minLength))
+        if field.maxLength is not None:
+            checks.append(check_field_max_length(field_name, field.maxLength))
+        if field.minimum is not None:
+            checks.append(check_field_minimum(field_name, field.minimum))
+        if field.maximum is not None:
+            checks.append(check_field_maximum(field_name, field.maximum))
+        if field.exclusiveMinimum is not None:
+            checks.append(check_field_minimum(field_name, field.exclusiveMinimum))
+            checks.append(check_field_not_equal(field_name, field.exclusiveMinimum))
+        if field.exclusiveMaximum is not None:
+            checks.append(check_field_maximum(field_name, field.exclusiveMaximum))
+            checks.append(check_field_not_equal(field_name, field.exclusiveMaximum))
+        if field.pattern is not None:
+            checks.append(check_field_regex(field_name, field.pattern))
+        if field.enum is not None and len(field.enum) > 0:
+            checks.append(check_field_enum(field_name, field.enum))
+        # TODO references: str = None
+        # TODO format
 
     return f"checks for {model_key}", checks
 
@@ -37,10 +64,8 @@ def check_field_is_present(field_name):
         "schema": {
             "name": f"Check that field {field_name} is present",
             "fail": {
-                "when required column missing": [
-                    field_name
-                ],
-            }
+                "when required column missing": [field_name],
+            },
         }
     }
 
@@ -49,27 +74,99 @@ def check_field_type(field_name: str, type: str):
     return {
         "schema": {
             "name": f"Check that field {field_name} has type {type}",
-            "fail": {
-                "when wrong column type": {
-                    field_name: type
-                }
-            }
+            "fail": {"when wrong column type": {field_name: type}},
+        }
+    }
+
+
+def check_field_required(field_name: str, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+
+    return {f"missing_count({field_name}) = 0": {"name": f"Check that required field {field_name} has no null values"}}
+
+
+def check_field_unique(field_name, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"duplicate_count({field_name}) = 0": {"name": f"Check that unique field {field_name} has no duplicate values"}
+    }
+
+
+def check_field_min_length(field_name, min_length, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a min length of {min}",
+            "valid min length": min_length,
+        }
+    }
+
+
+def check_field_max_length(field_name, max_length, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a max length of {max_length}",
+            "valid max length": max_length,
+        }
+    }
+
+
+def check_field_minimum(field_name, minimum, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a minimum of {min}",
+            "valid min": minimum,
+        }
+    }
+
+
+def check_field_maximum(field_name, maximum, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} has a maximum of {maximum}",
+            "valid max": maximum,
+        }
+    }
+
+
+def check_field_not_equal(field_name, value, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
+    return {
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} is not equal to {value}",
+            "invalid values": [value],
         }
     }
 
 
-def check_field_required(field_name):
+def check_field_enum(field_name, enum, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
     return {
-        f"missing_count({field_name}) = 0": {
-            "name": f"Check that required field {field_name} has no null values"
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} only contains enum values {enum}",
+            "valid values": enum,
         }
     }
 
 
-def check_field_unique(field_name):
+def check_field_regex(field_name, pattern, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f'"{field_name}"'
     return {
-        f'duplicate_count({field_name}) = 0': {
-            "name": f"Check that unique field {field_name} has no duplicate values"
+        f"invalid_count({field_name}) = 0": {
+            "name": f"Check that field {field_name} matches regex pattern {pattern}",
+            "valid regex": pattern,
         }
     }
 

datacontract/export/sql_converter.py

@@ -1,9 +1,10 @@
 from datacontract.export.sql_type_converter import convert_to_sql_type
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model
+from datacontract.model.data_contract_specification import DataContractSpecification, Model
 
 
-def to_sql_query(data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake") -> str:
+def to_sql_query(
+    data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake"
+) -> str:
     if data_contract_spec is None:
         return ""
     if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
@@ -42,27 +43,39 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str =
     if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
         return ""
 
+    table_prefix = ""
+
     for server_name, server in iter(data_contract_spec.servers.items()):
-        if server.type == server_type:
-            break
         if server.type == "snowflake":
             server_type = "snowflake"
             break
         if server.type == "postgres":
             server_type = "postgres"
             break
+        if server.type == "databricks":
+            server_type = "databricks"
+            if server.catalog is not None and server.schema_ is not None:
+                table_prefix = server.catalog + "." + server.schema_ + "."
+            break
+        if server.type == server_type:
+            break
 
     result = ""
     result += f"-- Data Contract: {data_contract_spec.id}\n"
     result += f"-- SQL Dialect: {server_type}\n"
     for model_name, model in iter(data_contract_spec.models.items()):
-        result += _to_sql_table(model_name, model, server_type)
+        result += _to_sql_table(table_prefix + model_name, model, server_type)
 
     return result.strip()
 
 
 def _to_sql_table(model_name, model, server_type="snowflake"):
-    result = f"CREATE TABLE {model_name} (\n"
+    if server_type == "databricks":
+        # Databricks recommends to use the CREATE OR REPLACE statement for unity managed tables
+        # https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html
+        result = f"CREATE OR REPLACE TABLE {model_name} (\n"
+    else:
+        result = f"CREATE TABLE {model_name} (\n"
     fields = len(model.fields)
     current_field_index = 1
     for field_name, field in iter(model.fields.items()):
@@ -72,11 +85,20 @@ def _to_sql_table(model_name, model, server_type="snowflake"):
             result += " not null"
         if field.primary:
             result += " primary key"
+        if server_type == "databricks" and field.description is not None:
+            result += f' COMMENT "{_escape(field.description)}"'
         if current_field_index < fields:
             result += ","
         result += "\n"
         current_field_index += 1
-    result += ");\n"
+    result += ")"
+    if server_type == "databricks" and model.description is not None:
+        result += f' COMMENT "{_escape(model.description)}"'
+    result += ";\n"
     return result
 
 
+def _escape(text: str | None) -> str | None:
+    if text is None:
+        return None
+    return text.replace('"', '\\"')

datacontract/export/sql_type_converter.py

@@ -6,7 +6,10 @@ def convert_to_sql_type(field: Field, server_type: str) -> str:
         return convert_to_snowflake(field)
     if server_type == "postgres":
         return convert_type_to_postgres(field)
-    return str(type)
+    if server_type == "databricks":
+        return convert_to_databricks(field)
+    return field.type
+
 
 # snowflake data types:
 # https://docs.snowflake.com/en/sql-reference/data-types.html
@@ -48,17 +51,16 @@ def convert_to_snowflake(field) -> None | str:
     return None
 
 
-
 # https://www.postgresql.org/docs/current/datatype.html
 # Using the name whenever possible
-def convert_type_to_postgres(field : Field) -> None | str:
+def convert_type_to_postgres(field: Field) -> None | str:
     type = field.type
     if type is None:
         return None
     if type.lower() in ["string", "varchar", "text"]:
         if field.format == "uuid":
             return "uuid"
-        return "text" # STRING does not exist, TEXT and VARCHAR are all the same in postrges
+        return "text"  # STRING does not exist, TEXT and VARCHAR are all the same in postrges
     if type.lower() in ["timestamp", "timestamp_tz"]:
         return "timestamptz"
     if type.lower() in ["timestamp_ntz"]:
@@ -89,3 +91,41 @@ def convert_type_to_postgres(field : Field) -> None | str:
     if type.lower() in ["array"]:
         return convert_to_sql_type(field.items, "postgres") + "[]"
     return None
+
+
+# databricks data types:
+# https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
+def convert_to_databricks(field) -> None | str:
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "STRING"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    if type.lower() in ["timestamp_ntz"]:
+        return "TIMESTAMP_NTZ"
+    if type.lower() in ["date"]:
+        return "DATE"
+    if type.lower() in ["time"]:
+        return "STRING"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        # precision and scale not supported by data contract
+        return "DECIMAL"
+    if type.lower() in ["float"]:
+        return "FLOAT"
+    if type.lower() in ["double"]:
+        return "DOUBLE"
+    if type.lower() in ["integer", "int"]:
+        return "INT"
+    if type.lower() in ["long", "bigint"]:
+        return "BIGINT"
+    if type.lower() in ["boolean"]:
+        return "BOOLEAN"
+    if type.lower() in ["object", "record", "struct"]:
+        return "STRUCT"
+    if type.lower() in ["bytes"]:
+        return "BINARY"
+    if type.lower() in ["array"]:
+        return "ARRAY"
+    return None

datacontract/export/terraform_converter.py

@@ -1,6 +1,7 @@
+import re
+
 from datacontract.model.data_contract_specification import \
     DataContractSpecification, Server
-import re
 
 
 def to_terraform(data_contract_spec: DataContractSpecification, server_id: str = None) -> str:
@@ -18,7 +19,7 @@ def to_terraform(data_contract_spec: DataContractSpecification, server_id: str =
     return result.strip()
 
 
-def server_to_terraform_resource(data_contract_spec, result, server : Server, server_name):
+def server_to_terraform_resource(data_contract_spec, result, server: Server, server_name):
     tag_data_contract = data_contract_spec.id
     tag_name = data_contract_spec.info.title
     tag_server = server_name
@@ -60,7 +61,7 @@ def extract_bucket_name(server) -> str | None:
     if server.type == "s3":
         s3_url = server.location
         # Regular expression to match the S3 bucket name
-        match = re.search(r's3://([^/]+)/', s3_url)
+        match = re.search(r"s3://([^/]+)/", s3_url)
         if match:
             # Return the first group (bucket name)
             return match.group(1)