datacontract-cli 0.9.7__py3-none-any.whl → 0.9.8__py3-none-any.whl

datacontract/export/dbt_converter.py

@@ -7,9 +7,6 @@ from datacontract.model.data_contract_specification import \
     DataContractSpecification, Model, Field
 
 
-
-
-
 def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
     dbt = {
         "version": 2,
@@ -23,7 +20,7 @@ def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
 
 def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name: str, model_value: Model) -> str:
     if data_contract_spec.models is None or len(data_contract_spec.models.items()) != 1:
-        print(f"Export to dbt-staging-sql currently only works with exactly one model in the data contract.")
+        print("Export to dbt-staging-sql currently only works with exactly one model in the data contract.")
         return ""
 
     id = data_contract_spec.id
@@ -39,15 +36,10 @@ def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name
 
 
 def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: str = None):
-    source = {
-        "name": data_contract_spec.id,
-        "tables": []
-    }
+    source = {"name": data_contract_spec.id, "tables": []}
     dbt = {
         "version": 2,
-        "sources": [
-            source
-        ],
+        "sources": [source],
     }
     if data_contract_spec.info.owner is not None:
         source["meta"] = {"owner": data_contract_spec.info.owner}
@@ -82,20 +74,14 @@ def _to_dbt_model(model_key, model_value: Model, data_contract_spec: DataContrac
         "name": model_key,
     }
     model_type = _to_dbt_model_type(model_value.type)
-    dbt_model["config"] = {
-        "meta": {
-            "data_contract": data_contract_spec.id
-        }
-    }
+    dbt_model["config"] = {"meta": {"data_contract": data_contract_spec.id}}
     dbt_model["config"]["materialized"] = model_type
 
     if data_contract_spec.info.owner is not None:
         dbt_model["config"]["meta"]["owner"] = data_contract_spec.info.owner
 
     if _supports_constraints(model_type):
-        dbt_model["config"]["contract"] = {
-            "enforced": True
-        }
+        dbt_model["config"]["contract"] = {"enforced": True}
     if model_value.description is not None:
         dbt_model["description"] = model_value.description
     columns = _to_columns(model_value.fields, _supports_constraints(model_type), True)
@@ -138,8 +124,8 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
             column["data_type"] = dbt_type
         else:
             column.setdefault("tests", []).append(
-                {"dbt_expectations.dbt_expectations.expect_column_values_to_be_of_type": {
-                    "column_type": dbt_type}})
+                {"dbt_expectations.dbt_expectations.expect_column_values_to_be_of_type": {"column_type": dbt_type}}
+            )
     if field.description is not None:
         column["description"] = field.description
     if field.required:
@@ -161,7 +147,8 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
         if field.maxLength is not None:
             length_test["max_value"] = field.maxLength
         column.setdefault("tests", []).append(
-            {"dbt_expectations.expect_column_value_lengths_to_be_between": length_test})
+            {"dbt_expectations.expect_column_value_lengths_to_be_between": length_test}
+        )
     if field.pii is not None:
         column.setdefault("meta", {})["pii"] = field.pii
     if field.classification is not None:
@@ -171,15 +158,26 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
     if field.pattern is not None:
         # Beware, the data contract pattern is a regex, not a like pattern
         column.setdefault("tests", []).append(
-            {"dbt_expectations.expect_column_values_to_match_regex": {"regex": field.pattern}})
-    if field.minimum is not None or field.maximum is not None and field.exclusiveMinimum is None and field.exclusiveMaximum is None:
+            {"dbt_expectations.expect_column_values_to_match_regex": {"regex": field.pattern}}
+        )
+    if (
+        field.minimum is not None
+        or field.maximum is not None
+        and field.exclusiveMinimum is None
+        and field.exclusiveMaximum is None
+    ):
         range_test = {}
         if field.minimum is not None:
             range_test["min_value"] = field.minimum
         if field.maximum is not None:
             range_test["max_value"] = field.maximum
         column.setdefault("tests", []).append({"dbt_expectations.expect_column_values_to_be_between": range_test})
-    elif field.exclusiveMinimum is not None or field.exclusiveMaximum is not None and field.minimum is None and field.maximum is None:
+    elif (
+        field.exclusiveMinimum is not None
+        or field.exclusiveMaximum is not None
+        and field.minimum is None
+        and field.maximum is None
+    ):
         range_test = {}
         if field.exclusiveMinimum is not None:
             range_test["min_value"] = field.exclusiveMinimum
@@ -190,17 +188,30 @@ def _to_column(field: Field, supports_constraints: bool, supports_datatype: bool
     else:
         if field.minimum is not None:
             column.setdefault("tests", []).append(
-                {"dbt_expectations.expect_column_values_to_be_between": {"min_value": field.minimum}})
+                {"dbt_expectations.expect_column_values_to_be_between": {"min_value": field.minimum}}
+            )
         if field.maximum is not None:
             column.setdefault("tests", []).append(
-                {"dbt_expectations.expect_column_values_to_be_between": {"max_value": field.maximum}})
+                {"dbt_expectations.expect_column_values_to_be_between": {"max_value": field.maximum}}
+            )
         if field.exclusiveMinimum is not None:
-            column.setdefault("tests", []).append({"dbt_expectations.expect_column_values_to_be_between": {
-                "min_value": field.exclusiveMinimum, "strictly": True}})
+            column.setdefault("tests", []).append(
+                {
+                    "dbt_expectations.expect_column_values_to_be_between": {
+                        "min_value": field.exclusiveMinimum,
+                        "strictly": True,
+                    }
+                }
+            )
         if field.exclusiveMaximum is not None:
-            column.setdefault("tests", []).append({"dbt_expectations.expect_column_values_to_be_between": {
-                "max_value": field.exclusiveMaximum, "strictly": True}})
+            column.setdefault("tests", []).append(
+                {
+                    "dbt_expectations.expect_column_values_to_be_between": {
+                        "max_value": field.exclusiveMaximum,
+                        "strictly": True,
+                    }
+                }
+            )
 
     # TODO: all constraints
     return column
-

datacontract/export/great_expectations_converter.py

@@ -0,0 +1,141 @@
+import json
+from typing import Dict, List, Any
+
+import yaml
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Field, Quality
+
+
+def to_great_expectations(data_contract_spec: DataContractSpecification, model_key: str) -> str:
+    """
+    Convert each model in the contract to a Great Expectation suite
+    @param data_contract_spec: data contract to export to great expectations
+    @param model_key: model to great expectations to
+    @return: a dictionary of great expectation suites
+    """
+    expectations = []
+    model_value = data_contract_spec.models.get(model_key)
+    quality_checks = get_quality_checks(data_contract_spec.quality)
+    expectations.extend(model_to_expectations(model_value.fields))
+    expectations.extend(checks_to_expectations(quality_checks, model_key))
+    model_expectation_suite = to_suite(model_key, data_contract_spec.info.version, expectations)
+
+    return model_expectation_suite
+
+
+def to_suite(
+    model_key: str,
+    contract_version: str,
+    expectations: List[Dict[str, Any]],
+) -> str:
+    return json.dumps(
+        {
+            "data_asset_type": "null",
+            "expectation_suite_name": "user-defined.{model_key}.{contract_version}".format(
+                model_key=model_key, contract_version=contract_version
+            ),
+            "expectations": expectations,
+            "meta": {},
+        },
+        indent=2,
+    )
+
+
+def model_to_expectations(fields: Dict[str, Field]) -> List[Dict[str, Any]]:
+    """
+    Convert the model information to expectations
+    @param fields: model field
+    @return: list of expectations
+    """
+    expectations = []
+    add_column_order_exp(fields, expectations)
+    for field_name, field in fields.items():
+        add_field_expectations(field_name, field, expectations)
+    return expectations
+
+
+def add_field_expectations(field_name, field: Field, expectations: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    if field.type is not None:
+        expectations.append(to_column_types_exp(field_name, field.type))
+    if field.unique is not None:
+        expectations.append(to_column_unique_exp(field_name))
+    if field.maxLength is not None or field.minLength is not None:
+        expectations.append(to_column_length_exp(field_name, field.minLength, field.maxLength))
+    if field.minimum is not None or field.maximum is not None:
+        expectations.append(to_column_min_max_exp(field_name, field.minimum, field.maximum))
+
+    # TODO: all constraints
+    return expectations
+
+
+def add_column_order_exp(fields: Dict[str, Field], expectations: List[Dict[str, Any]]):
+    expectations.append(
+        {
+            "expectation_type": "expect_table_columns_to_match_ordered_list",
+            "kwargs": {"column_list": list(fields.keys())},
+            "meta": {},
+        }
+    )
+
+
+def to_column_types_exp(field_name, field_type) -> Dict[str, Any]:
+    return {
+        "expectation_type": "expect_column_values_to_be_of_type",
+        "kwargs": {"column": field_name, "type_": field_type},
+        "meta": {},
+    }
+
+
+def to_column_unique_exp(field_name) -> Dict[str, Any]:
+    return {"expectation_type": "expect_column_values_to_be_unique", "kwargs": {"column": field_name}, "meta": {}}
+
+
+def to_column_length_exp(field_name, min_length, max_length) -> Dict[str, Any]:
+    return {
+        "expectation_type": "expect_column_value_lengths_to_be_between",
+        "kwargs": {"column": field_name, "min_value": min_length, "max_value": max_length},
+        "meta": {},
+    }
+
+
+def to_column_min_max_exp(field_name, minimum, maximum) -> Dict[str, Any]:
+    return {
+        "expectation_type": "expect_column_values_to_be_between",
+        "kwargs": {"column": field_name, "min_value": minimum, "max_value": maximum},
+        "meta": {},
+    }
+
+
+def get_quality_checks(quality: Quality) -> Dict[str, Any]:
+    if quality is None:
+        return {}
+    if quality.type is None:
+        return {}
+    if quality.type.lower() != "great-expectations":
+        return {}
+    if isinstance(quality.specification, str):
+        quality_specification = yaml.safe_load(quality.specification)
+    else:
+        quality_specification = quality.specification
+    return quality_specification
+
+
+def checks_to_expectations(quality_checks: Dict[str, Any], model_key: str) -> List[Dict[str, Any]]:
+    """
+    Get the quality definition for each model to the model expectation list
+    @param quality_checks: dictionary of quality checks by model
+    @param model_key: id of the model
+    @return: the list of expectations for that model
+    """
+    if quality_checks is None or model_key not in quality_checks:
+        return []
+
+    model_quality_checks = quality_checks[model_key]
+
+    if model_quality_checks is None:
+        return []
+
+    if isinstance(model_quality_checks, str):
+        expectation_list = json.loads(model_quality_checks)
+        return expectation_list

datacontract/export/jsonschema_converter.py

@@ -12,16 +12,18 @@ def to_jsonschemas(data_contract_spec: DataContractSpecification):
         jsonschmemas[model_key] = jsonschema
     return jsonschmemas
 
+
 def to_jsonschema_json(model_key, model_value: Model) -> str:
     jsonschema = to_jsonschema(model_key, model_value)
     return json.dumps(jsonschema, indent=2)
 
+
 def to_jsonschema(model_key, model_value: Model) -> dict:
     return {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "type": "object",
         "properties": to_properties(model_value.fields),
-        "required": to_required(model_value.fields)
+        "required": to_required(model_value.fields),
     }
 
 

datacontract/export/odcs_converter.py

@@ -27,10 +27,12 @@ def to_odcs_yaml(data_contract_spec: DataContractSpecification):
         odcs["description"] = {
             "purpose": None,
             "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
-            "limitations": data_contract_spec.terms.limitations.strip() if data_contract_spec.terms.limitations is not None else None,
+            "limitations": data_contract_spec.terms.limitations.strip()
+            if data_contract_spec.terms.limitations is not None
+            else None,
         }
 
-    odcs["type"] = "tables" # required, TODO read from models.type?
+    odcs["type"] = "tables"  # required, TODO read from models.type?
     odcs["dataset"] = []
 
     for model_key, model_value in data_contract_spec.models.items():
@@ -62,9 +64,7 @@ def to_columns(fields: Dict[str, Field]) -> list:
 
 
 def to_column(field_name: str, field: Field) -> dict:
-    column = {
-        "column": field_name
-    }
+    column = {"column": field_name}
     if field.type is not None:
         column["logicalType"] = field.type
         column["physicalType"] = field.type
@@ -100,5 +100,3 @@ def to_column(field_name: str, field: Field) -> dict:
 
     # todo enum
     return column
-
-

datacontract/export/protobuf_converter.py

@@ -1,9 +1,4 @@
-from typing import Dict
-
-import yaml
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 def to_protobuf(data_contract_spec: DataContractSpecification):
@@ -24,7 +19,7 @@ def _to_protobuf_message_name(model_name):
     return model_name[0].upper() + model_name[1:]
 
 
-def to_protobuf_message(model_name, fields, description, indent_level:int = 0):
+def to_protobuf_message(model_name, fields, description, indent_level: int = 0):
     result = ""
 
     if description is not None:
@@ -34,8 +29,15 @@ def to_protobuf_message(model_name, fields, description, indent_level:int = 0):
     number = 1
     for field_name, field in fields.items():
         if field.type in ["object", "record", "struct"]:
-            fields_protobuf += "\n".join(
-                map(lambda x: "  " + x, to_protobuf_message(field_name, field.fields, field.description, indent_level + 1).splitlines())) + "\n"
+            fields_protobuf += (
+                "\n".join(
+                    map(
+                        lambda x: "  " + x,
+                        to_protobuf_message(field_name, field.fields, field.description, indent_level + 1).splitlines(),
+                    )
+                )
+                + "\n"
+            )
 
         fields_protobuf += to_protobuf_field(field_name, field, field.description, number, 1) + "\n"
         number += 1
@@ -44,7 +46,7 @@ def to_protobuf_message(model_name, fields, description, indent_level:int = 0):
     return result
 
 
-def to_protobuf_field(field_name, field, description, number:int, indent_level:int = 0):
+def to_protobuf_field(field_name, field, description, number: int, indent_level: int = 0):
     optional = ""
     if not field.required:
         optional = "optional "

datacontract/export/pydantic_converter.py

@@ -0,0 +1,140 @@
+import datacontract.model.data_contract_specification as spec
+import typing
+import ast
+
+def to_pydantic_model_str(contract: spec.DataContractSpecification) -> str:
+    classdefs = [generate_model_class(model_name, model) for (model_name, model) in contract.models.items()]
+    documentation = [ast.Expr(ast.Constant(contract.info.description))] if (
+        contract.info and contract.info.description) else []
+    result = ast.Module(body=[
+        ast.Import(
+            names=[ast.Name("datetime", ctx=ast.Load()),
+                   ast.Name("typing", ctx=ast.Load()),
+                   ast.Name("pydantic", ctx=ast.Load())]),
+        *documentation,
+        *classdefs],
+        type_ignores=[])
+    return ast.unparse(result)
+
+def optional_of(node) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Attribute(
+            ast.Name(id="typing", ctx=ast.Load()),
+            attr="Optional",
+            ctx=ast.Load()),
+        slice=node)
+
+def list_of(node) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Name(id="list", ctx=ast.Load()),
+        slice=node)
+
+def product_of(nodes: list[typing.Any]) -> ast.Subscript:
+    return ast.Subscript(
+        value=ast.Attribute(
+            value=ast.Name(id="typing", ctx=ast.Load()),
+            attr="Product",
+            ctx=ast.Load()),
+        slice=ast.Tuple(nodes, ctx=ast.Load())
+    )
+
+
+type_annotation_type = typing.Union[ast.Name, ast.Attribute, ast.Constant, ast.Subscript]
+
+def constant_field_annotation(field_name: str, field: spec.Field)\
+        -> tuple[type_annotation_type,
+                 typing.Optional[ast.ClassDef]]:
+    match field.type:
+        case "string"|"text"|"varchar":
+            return (ast.Name("str", ctx=ast.Load()), None)
+        case "number", "decimal", "numeric":
+            # Either integer or float in specification,
+            # so we use float.
+            return (ast.Name("float", ctx=ast.Load()), None)
+        case "int" | "integer" | "long" | "bigint":
+            return (ast.Name("int", ctx=ast.Load()), None)
+        case "float" | "double":
+            return (ast.Name("float", ctx=ast.Load()), None)
+        case "boolean":
+            return (ast.Name("bool", ctx=ast.Load()), None)
+        case "timestamp" | "timestamp_tz" | "timestamp_ntz":
+            return (ast.Attribute(
+                value=ast.Name(id="datetime", ctx=ast.Load()),
+                attr="datetime"), None)
+        case "date":
+            return (ast.Attribute(
+                value=ast.Name(id="datetime", ctx=ast.Load()),
+                attr="date"), None)
+        case "bytes":
+            return (ast.Name("bytes", ctx=ast.Load()), None)
+        case "null":
+            return (ast.Constant("None"), None)
+        case "array":
+            (annotated_type, new_class) = type_annotation(field_name, field.items)
+            return (list_of(annotated_type), new_class)
+        case "object" | "record" | "struct":
+            classdef = generate_field_class(field_name.capitalize(), field)
+            return (ast.Name(field_name.capitalize(), ctx=ast.Load()), classdef)
+        case _:
+            raise RuntimeError(f"Unsupported field type {field.type}.")
+
+
+def type_annotation(field_name: str, field: spec.Field) -> tuple[type_annotation_type, typing.Optional[ast.ClassDef]]:
+    if field.required:
+        return constant_field_annotation(field_name, field)
+    else:
+        (annotated_type, new_classes) = constant_field_annotation(field_name, field)
+        return (optional_of(annotated_type), new_classes)
+
+def is_simple_field(field: spec.Field) -> bool:
+    return field.type not in set(["object", "record", "struct"])
+
+def field_definitions(fields: dict[str, spec.Field]) ->\
+        tuple[list[ast.Expr],
+              list[ast.ClassDef]]:
+    annotations = []
+    classes = []
+    for (field_name, field) in fields.items():
+        (ann, new_class) = type_annotation(field_name, field)
+        annotations.append(
+            ast.AnnAssign(
+                target=ast.Name(id=field_name, ctx=ast.Store()),
+                annotation=ann,
+                simple=1))
+        if field.description and is_simple_field(field):
+            annotations.append(
+                ast.Expr(ast.Constant(field.description)))
+        if new_class:
+            classes.append(new_class)
+    return (annotations, classes)
+
+def generate_field_class(field_name: str, field: spec.Field) -> ast.ClassDef:
+    assert(field.type in set(["object", "record", "struct"]))
+    (annotated_type, new_classes) = field_definitions(field.fields)
+    documentation = [ast.Expr(ast.Constant(field.description))] if field.description else []
+    return ast.ClassDef(
+        name=field_name,
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()),
+                             attr="BaseModel",
+                             ctx=ast.Load())],
+        body=[
+            *documentation,
+            *new_classes,
+            *annotated_type
+        ],
+        keywords=[],
+        decorator_list=[])
+
+
+def generate_model_class(name: str, model_definition: spec.Model) -> ast.ClassDef:
+    (field_assignments, nested_classes) = field_definitions(model_definition.fields)
+    documentation = [ast.Expr(ast.Constant(model_definition.description))] if model_definition.description else []
+    result = ast.ClassDef(
+        name=name.capitalize(),
+        bases=[ast.Attribute(value=ast.Name(id="pydantic", ctx=ast.Load()),
+                             attr="BaseModel",
+                             ctx=ast.Load())],
+        body=[*documentation, *nested_classes, *field_assignments],
+        keywords=[],
+        decorator_list=[])
+    return result

datacontract/export/rdf_converter.py

@@ -1,17 +1,39 @@
-from typing import Dict
-import inspect
 from pydantic import BaseModel
 from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
 
 from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model, Field
+    DataContractSpecification
 
 
 def is_literal(property_name):
-    return property_name in ["dataContractSpecification", "title", "version", "description", "name", "url", "type",
-                             "location", "format", "delimiter", "usage", "limitations",
-                             "billing", "noticePeriod", "required", "unique", "minLength", "maxLength", "example",
-                             "pii", "classification", "data", "enum", "minimum", "maximum", "patterns"]
+    return property_name in [
+        "dataContractSpecification",
+        "title",
+        "version",
+        "description",
+        "name",
+        "url",
+        "type",
+        "location",
+        "format",
+        "delimiter",
+        "usage",
+        "limitations",
+        "billing",
+        "noticePeriod",
+        "required",
+        "unique",
+        "minLength",
+        "maxLength",
+        "example",
+        "pii",
+        "classification",
+        "data",
+        "enum",
+        "minimum",
+        "maximum",
+        "patterns",
+    ]
 
 
 def is_uriref(property_name):
@@ -21,6 +43,7 @@ def is_uriref(property_name):
 def to_rdf_n3(data_contract_spec: DataContractSpecification, base) -> str:
     return to_rdf(data_contract_spec, base).serialize(format="n3")
 
+
 def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
     if base is not None:
         g = Graph(base=base)
@@ -61,7 +84,7 @@ def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
 
 def add_example(contract, example, graph, dc, dcx):
     an_example = BNode()
-    graph.add((contract, dc['example'], an_example))
+    graph.add((contract, dc["example"], an_example))
     graph.add((an_example, RDF.type, URIRef(dc + "Example")))
     for example_property in example.model_fields:
         add_triple(sub=an_example, pred=example_property, obj=example, graph=graph, dc=dc, dcx=dcx)
@@ -81,14 +104,14 @@ def add_triple(sub, pred, obj, graph, dc, dcx):
 
 def add_model(contract, model, model_name, graph, dc, dcx):
     a_model = URIRef(model_name)
-    graph.add((contract, dc['model'], a_model))
+    graph.add((contract, dc["model"], a_model))
     graph.add((a_model, dc.description, Literal(model.description)))
     graph.add((a_model, RDF.type, URIRef(dc + "Model")))
     for field_name, field in model.fields.items():
         a_field = BNode()
-        graph.add((a_model, dc['field'], a_field))
+        graph.add((a_model, dc["field"], a_field))
         graph.add((a_field, RDF.type, URIRef(dc + "Field")))
-        graph.add((a_field, dc['name'], Literal(field_name)))
+        graph.add((a_field, dc["name"], Literal(field_name)))
         for field_property in field.model_fields:
             add_triple(sub=a_field, pred=field_property, obj=field, graph=graph, dc=dc, dcx=dcx)