datacontract-cli 0.9.6.post2__py3-none-any.whl → 0.9.8__py3-none-any.whl

datacontract/export/odcs_converter.py

@@ -27,16 +27,18 @@ def to_odcs_yaml(data_contract_spec: DataContractSpecification):
         odcs["description"] = {
             "purpose": None,
             "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
-            "limitations": data_contract_spec.terms.limitations.strip() if data_contract_spec.terms.limitations is not None else None,
+            "limitations": data_contract_spec.terms.limitations.strip()
+            if data_contract_spec.terms.limitations is not None
+            else None,
         }
 
-    odcs["type"] = "tables" # required, TODO read from models.type?
+    odcs["type"] = "tables"  # required, TODO read from models.type?
     odcs["dataset"] = []
 
     for model_key, model_value in data_contract_spec.models.items():
         odcs_table = to_odcs_table(model_key, model_value)
         odcs["dataset"].append(odcs_table)
-    return yaml.dump(odcs, indent=2, sort_keys=False)
+    return yaml.dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
 
 
 def to_odcs_table(model_key, model_value: Model) -> dict:
@@ -62,9 +64,7 @@ def to_columns(fields: Dict[str, Field]) -> list:
 
 
 def to_column(field_name: str, field: Field) -> dict:
-    column = {
-        "column": field_name
-    }
+    column = {"column": field_name}
     if field.type is not None:
         column["logicalType"] = field.type
         column["physicalType"] = field.type
@@ -91,14 +91,12 @@ def to_column(field_name: str, field: Field) -> dict:
         column["tags"].append(f"minimum:{field.minimum}")
     if field.maximum is not None:
         column["tags"].append(f"maximum:{field.maximum}")
-    if field.minimumExclusive is not None:
-        column["tags"].append(f"minimumExclusive:{field.minimumExclusive}")
-    if field.maximumExclusive is not None:
-        column["tags"].append(f"maximumExclusive:{field.maximumExclusive}")
+    if field.exclusiveMinimum is not None:
+        column["tags"].append(f"exclusiveMinimum:{field.exclusiveMinimum}")
+    if field.exclusiveMaximum is not None:
+        column["tags"].append(f"exclusiveMaximum:{field.exclusiveMaximum}")
     if not column["tags"]:
         del column["tags"]
 
     # todo enum
     return column
-
-

datacontract/export/protobuf_converter.py

@@ -0,0 +1,99 @@
+from datacontract.model.data_contract_specification import DataContractSpecification
+
+
+def to_protobuf(data_contract_spec: DataContractSpecification):
+    messages = ""
+    for model_name, model in data_contract_spec.models.items():
+        messages += to_protobuf_message(model_name, model.fields, model.description, 0)
+        messages += "\n"
+
+    result = f"""syntax = "proto3";
+
+{messages}
+"""
+
+    return result
+
+
+def _to_protobuf_message_name(model_name):
+    return model_name[0].upper() + model_name[1:]
+
+
+def to_protobuf_message(model_name, fields, description, indent_level: int = 0):
+    result = ""
+
+    if description is not None:
+        result += f"""{indent(indent_level)}/* {description} */\n"""
+
+    fields_protobuf = ""
+    number = 1
+    for field_name, field in fields.items():
+        if field.type in ["object", "record", "struct"]:
+            fields_protobuf += (
+                "\n".join(
+                    map(
+                        lambda x: "  " + x,
+                        to_protobuf_message(field_name, field.fields, field.description, indent_level + 1).splitlines(),
+                    )
+                )
+                + "\n"
+            )
+
+        fields_protobuf += to_protobuf_field(field_name, field, field.description, number, 1) + "\n"
+        number += 1
+    result += f"message {_to_protobuf_message_name(model_name)} {{\n{fields_protobuf}}}\n"
+
+    return result
+
+
+def to_protobuf_field(field_name, field, description, number: int, indent_level: int = 0):
+    optional = ""
+    if not field.required:
+        optional = "optional "
+
+    result = ""
+
+    if description is not None:
+        result += f"""{indent(indent_level)}/* {description} */\n"""
+
+    result += f"{indent(indent_level)}{optional}{_convert_type(field_name, field)} {field_name} = {number};"
+
+    return result
+
+
+def indent(indent_level):
+    return "  " * indent_level
+
+
+def _convert_type(field_name, field) -> None | str:
+    type = field.type
+    if type is None:
+        return None
+    if type.lower() in ["string", "varchar", "text"]:
+        return "string"
+    if type.lower() in ["timestamp", "timestamp_tz"]:
+        return "string"
+    if type.lower() in ["timestamp_ntz"]:
+        return "string"
+    if type.lower() in ["date"]:
+        return "string"
+    if type.lower() in ["time"]:
+        return "string"
+    if type.lower() in ["number", "decimal", "numeric"]:
+        return "double"
+    if type.lower() in ["float", "double"]:
+        return type.lower()
+    if type.lower() in ["integer", "int"]:
+        return "int32"
+    if type.lower() in ["long", "bigint"]:
+        return "int64"
+    if type.lower() in ["boolean"]:
+        return "bool"
+    if type.lower() in ["bytes"]:
+        return "bytes"
+    if type.lower() in ["object", "record", "struct"]:
+        return _to_protobuf_message_name(field_name)
+    if type.lower() in ["array"]:
+        # TODO spec is missing arrays
+        return "repeated string"
+    return None

datacontract/export/pydantic_converter.py

@@ -0,0 +1,140 @@
+import datacontract.model.data_contract_specification as spec
+import typing
+import ast
+
+def to_pydantic_model_str(contract: spec.DataContractSpecification) -> str:
+    classdefs = [generate_model_class(model_name, model) for (model_name, model) in contract.models.items()]
+    documentation = [ast.Expr(ast.Constant(contract.info.description))] if (
+        contract.info and contract.info.description) else []
+    result = ast.Module(body=[
+        ast.Import(
+            names=[ast.Name("datetime", ctx=ast.Load()),
+                   ast.Name("typing", ctx=ast.Load()),
+                   ast.Name("pydantic", ctx=ast.Load())]),
+        *documentation,
+        *classdefs],
+        type_ignores=[])
+    return ast.unparse(result)
+
+def optional_of(node) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Attribute(
+            ast.Name(id="typing", ctx=ast.Load()),
+            attr="Optional",
+            ctx=ast.Load()),
+        slice=node)
+
+def list_of(node) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Name(id="list", ctx=ast.Load()),
+        slice=node)
+
+def product_of(nodes: list[typing.Any]) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Attribute(
+            value=ast.Name(id="typing", ctx=ast.Load()),
+            attr="Product",
+            ctx=ast.Load()),
+        slice=ast.Tuple(nodes, ctx=ast.Load())
+    )
+
+
+type_annotation_type = typing.Union[ast.Name, ast.Attribute, ast.Constant, ast.Subscript]
+
+def constant_field_annotation(field_name: str, field: spec.Field)\
+        -> tuple[type_annotation_type,
+                 typing.Optional[ast.ClassDef]]:
+    match field.type:
+        case "string"|"text"|"varchar":
+            return (ast.Name("str", ctx=ast.Load()), None)
+        case "number", "decimal", "numeric":
+            # Either integer or float in specification,
+            # so we use float.
+            return (ast.Name("float", ctx=ast.Load()), None)
+        case "int" | "integer" | "long" | "bigint":
+            return (ast.Name("int", ctx=ast.Load()), None)
+        case "float" | "double":
+            return (ast.Name("float", ctx=ast.Load()), None)
+        case "boolean":
+            return (ast.Name("bool", ctx=ast.Load()), None)
+        case "timestamp" | "timestamp_tz" | "timestamp_ntz":
+            return (ast.Attribute(
+                value=ast.Name(id="datetime", ctx=ast.Load()),
+                attr="datetime"), None)
+        case "date":
+            return (ast.Attribute(
+                value=ast.Name(id="datetime", ctx=ast.Load()),
+                attr="date"), None)
+        case "bytes":
+            return (ast.Name("bytes", ctx=ast.Load()), None)
+        case "null":
+            return (ast.Constant("None"), None)
+        case "array":
+            (annotated_type, new_class) = type_annotation(field_name, field.items)
+            return (list_of(annotated_type), new_class)
+        case "object" | "record" | "struct":
+            classdef = generate_field_class(field_name.capitalize(), field)
+            return (ast.Name(field_name.capitalize(), ctx=ast.Load()), classdef)
+        case _:
+            raise RuntimeError(f"Unsupported field type {field.type}.")
+
+
+def type_annotation(field_name: str, field: spec.Field) -> tuple[type_annotation_type, typing.Optional[ast.ClassDef]]:
+    if field.required:
+        return constant_field_annotation(field_name, field)
+    else:
+        (annotated_type, new_classes) = constant_field_annotation(field_name, field)
+        return (optional_of(annotated_type), new_classes)
+
+def is_simple_field(field: spec.Field) -> bool:
+    return field.type not in set(["object", "record", "struct"])
+
+def field_definitions(fields: dict[str, spec.Field]) ->\
+        tuple[list[ast.Expr],
+              list[ast.ClassDef]]:
+    annotations = []
+    classes = []
+    for (field_name, field) in fields.items():
+        (ann, new_class) = type_annotation(field_name, field)
+        annotations.append(
+            ast.AnnAssign(
+                target=ast.Name(id=field_name, ctx=ast.Store()),
+                annotation=ann,
+                simple=1))
+        if field.description and is_simple_field(field):
+            annotations.append(
+                ast.Expr(ast.Constant(field.description)))
+        if new_class:
+            classes.append(new_class)
+    return (annotations, classes)
+
+def generate_field_class(field_name: str, field: spec.Field) -> ast.ClassDef:
+    assert(field.type in set(["object", "record", "struct"]))
+    (annotated_type, new_classes) = field_definitions(field.fields)
+    documentation = [ast.Expr(ast.Constant(field.description))] if field.description else []
+    return ast.ClassDef(
+        name=field_name,
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()),
+                             attr="BaseModel",
+                             ctx=ast.Load())],
+        body=[
+            *documentation,
+            *new_classes,
+            *annotated_type
+        ],
+        keywords=[],
+        decorator_list=[])
+
+
+def generate_model_class(name: str, model_definition: spec.Model) -> ast.ClassDef:
+    (field_assignments, nested_classes) = field_definitions(model_definition.fields)
+    documentation = [ast.Expr(ast.Constant(model_definition.description))] if model_definition.description else []
+    result = ast.ClassDef(
+        name=name.capitalize(),
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()),
+                             attr="BaseModel",
+                             ctx=ast.Load())],
+        body=[*documentation, *nested_classes, *field_assignments],
+        keywords=[],
+        decorator_list=[])
+    return result

datacontract/export/rdf_converter.py

@@ -1,26 +1,49 @@
-from typing import Dict
-import inspect
 from pydantic import BaseModel
 from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
 
 from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
+    DataContractSpecification
 
 
 def is_literal(property_name):
-    return property_name in ["dataContractSpecification", "title", "version", "description", "name", "url", "type",
-                             "location", "format", "delimiter", "usage", "limitations",
-                             "billing", "noticePeriod", "required", "unique", "minLength", "maxLength", "example",
-                             "pii", "classification", "data", "enum", "minimum", "maximum", "patterns"]
+    return property_name in [
+        "dataContractSpecification",
+        "title",
+        "version",
+        "description",
+        "name",
+        "url",
+        "type",
+        "location",
+        "format",
+        "delimiter",
+        "usage",
+        "limitations",
+        "billing",
+        "noticePeriod",
+        "required",
+        "unique",
+        "minLength",
+        "maxLength",
+        "example",
+        "pii",
+        "classification",
+        "data",
+        "enum",
+        "minimum",
+        "maximum",
+        "patterns",
+    ]
 
 
 def is_uriref(property_name):
     return property_name in ["model", "domain", "owner"]
 
 
-def to_rdf_n3(data_contract_spec: DataContractSpecification, base) -> Graph:
+def to_rdf_n3(data_contract_spec: DataContractSpecification, base) -> str:
     return to_rdf(data_contract_spec, base).serialize(format="n3")
 
+
 def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
     if base is not None:
         g = Graph(base=base)
@@ -61,7 +84,7 @@ def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
 
 def add_example(contract, example, graph, dc, dcx):
     an_example = BNode()
-    graph.add((contract, dc['example'], an_example))
+    graph.add((contract, dc["example"], an_example))
     graph.add((an_example, RDF.type, URIRef(dc + "Example")))
     for example_property in example.model_fields:
         add_triple(sub=an_example, pred=example_property, obj=example, graph=graph, dc=dc, dcx=dcx)
@@ -81,14 +104,14 @@ def add_triple(sub, pred, obj, graph, dc, dcx):
 
 def add_model(contract, model, model_name, graph, dc, dcx):
     a_model = URIRef(model_name)
-    graph.add((contract, dc['model'], a_model))
+    graph.add((contract, dc["model"], a_model))
     graph.add((a_model, dc.description, Literal(model.description)))
     graph.add((a_model, RDF.type, URIRef(dc + "Model")))
     for field_name, field in model.fields.items():
         a_field = BNode()
-        graph.add((a_model, dc['field'], a_field))
+        graph.add((a_model, dc["field"], a_field))
         graph.add((a_field, RDF.type, URIRef(dc + "Field")))
-        graph.add((a_field, dc['name'], Literal(field_name)))
+        graph.add((a_field, dc["name"], Literal(field_name)))
         for field_property in field.model_fields:
             add_triple(sub=a_field, pred=field_property, obj=field, graph=graph, dc=dc, dcx=dcx)
 

datacontract/export/sodacl_converter.py

@@ -1,14 +1,15 @@
 import yaml
 
+from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import \
     DataContractSpecification
 
 
-def to_sodacl_yaml(data_contract_spec: DataContractSpecification, check_types: bool = True) -> str:
+def to_sodacl_yaml(data_contract_spec: DataContractSpecification, server_type: str = None, check_types: bool = True) -> str:
     try:
         sodacl = {}
         for model_key, model_value in data_contract_spec.models.items():
-            k, v = to_checks(model_key, model_value, check_types)
+            k, v = to_checks(model_key, model_value, server_type, check_types)
             sodacl[k] = v
         add_quality_checks(sodacl, data_contract_spec)
         sodacl_yaml_str = yaml.dump(sodacl, default_flow_style=False, sort_keys=False)
@@ -17,17 +18,21 @@ def to_sodacl_yaml(data_contract_spec: DataContractSpecification, check_types: b
         return f"Error: {e}"
 
 
-def to_checks(model_key, model_value, check_types: bool):
+def to_checks(model_key, model_value, server_type: str, check_types: bool):
     checks = []
     fields = model_value.fields
+
+    quote_field_name = server_type in ["postgres"]
+
     for field_name, field in fields.items():
         checks.append(check_field_is_present(field_name))
         if check_types and field.type is not None:
-            checks.append(check_field_type(field_name, field.type))
+            sql_type = convert_to_sql_type(field, server_type)
+            checks.append(check_field_type(field_name, sql_type))
         if field.required:
-            checks.append(check_field_required(field_name))
+            checks.append(check_field_required(field_name, quote_field_name))
         if field.unique:
-            checks.append(check_field_unique(field_name))
+            checks.append(check_field_unique(field_name, quote_field_name))
 
     return f"checks for {model_key}", checks
 
@@ -37,10 +42,8 @@ def check_field_is_present(field_name):
         "schema": {
             "name": f"Check that field {field_name} is present",
             "fail": {
-                "when required column missing": [
-                    field_name
-                ],
-            }
+                "when required column missing": [field_name],
+            },
         }
     }
 
@@ -49,28 +52,25 @@ def check_field_type(field_name: str, type: str):
     return {
         "schema": {
             "name": f"Check that field {field_name} has type {type}",
-            "fail": {
-                "when wrong column type": {
-                    field_name: type
-                }
-            }
+            "fail": {"when wrong column type": {field_name: type}},
         }
     }
 
 
-def check_field_required(field_name):
+def check_field_required(field_name: str, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f"\"{field_name}\""
+
     return {
-        f"missing_count({field_name}) = 0": {
-            "name": f"Check that required field {field_name} has no null values"
-        }
-    }
+        f"missing_count({field_name}) = 0": {"name": f"Check that required field {field_name} has no null values"}}
 
 
-def check_field_unique(field_name):
+def check_field_unique(field_name, quote_field_name: bool = False):
+    if quote_field_name:
+        field_name = f"\"{field_name}\""
     return {
-        f'duplicate_count({field_name}) = 0': {
-            "name": f"Check that unique field {field_name} has no duplicate values"
-        }
+        f"duplicate_count({field_name}) = 0": {
+            "name": f"Check that unique field {field_name} has no duplicate values"}
     }
 
 

datacontract/export/sql_converter.py

@@ -0,0 +1,93 @@
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.data_contract_specification import DataContractSpecification, Model
+
+
+def to_sql_query(
+    data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake"
+) -> str:
+    if data_contract_spec is None:
+        return ""
+    if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
+        return ""
+
+    result = ""
+    result += f"-- Data Contract: {data_contract_spec.id}\n"
+    result += f"-- SQL Dialect: {server_type}\n"
+    result += _to_sql_query(model_name, model_value, server_type)
+
+    return result
+
+
+def _to_sql_query(model_name, model_value, server_type) -> str:
+    columns = []
+    for field_name, field in model_value.fields.items():
+        # TODO escape SQL reserved key words, probably dependent on server type
+        columns.append(field_name)
+
+    result = "select\n"
+    current_column_index = 1
+    number_of_columns = len(columns)
+    for column in columns:
+        result += f"    {column}"
+        if current_column_index < number_of_columns:
+            result += ","
+        result += "\n"
+        current_column_index += 1
+    result += f"from {model_name}\n"
+    return result
+
+
+def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str = "snowflake") -> str:
+    if data_contract_spec is None:
+        return ""
+    if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
+        return ""
+
+    table_prefix = ""
+
+    for server_name, server in iter(data_contract_spec.servers.items()):
+        if server.type == "snowflake":
+            server_type = "snowflake"
+            break
+        if server.type == "postgres":
+            server_type = "postgres"
+            break
+        if server.type == "databricks":
+            server_type = "databricks"
+            if server.catalog is not None and server.schema_ is not None:
+                table_prefix = server.catalog + "." + server.schema_ + "."
+            break
+        if server.type == server_type:
+            break
+
+    result = ""
+    result += f"-- Data Contract: {data_contract_spec.id}\n"
+    result += f"-- SQL Dialect: {server_type}\n"
+    for model_name, model in iter(data_contract_spec.models.items()):
+        result += _to_sql_table(table_prefix + model_name, model, server_type)
+
+    return result.strip()
+
+
+def _to_sql_table(model_name, model, server_type="snowflake"):
+    if server_type == "databricks":
+        # Databricks recommends to use the CREATE OR REPLACE statement for unity managed tables
+        # https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html
+        result = f"CREATE OR REPLACE TABLE {model_name} (\n"
+    else:
+        result = f"CREATE TABLE {model_name} (\n"
+    fields = len(model.fields)
+    current_field_index = 1
+    for field_name, field in iter(model.fields.items()):
+        type = convert_to_sql_type(field, server_type)
+        result += f"  {field_name} {type}"
+        if field.required:
+            result += " not null"
+        if field.primary:
+            result += " primary key"
+        if current_field_index < fields:
+            result += ","
+        result += "\n"
+        current_field_index += 1
+    result += ");\n"
+    return result