datacontract-cli 0.10.7__py3-none-any.whl → 0.10.9__py3-none-any.whl

datacontract/export/avro_converter.py

@@ -1,8 +1,15 @@
 import json
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
 from datacontract.model.data_contract_specification import Field
 
 
+class AvroExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_avro_schema_json(model_name, model_value)
+
+
 def to_avro_schema(model_name, model) -> dict:
     return to_avro_record(model_name, model.fields, model.description, model.namespace)
 
@@ -47,7 +54,13 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         if "avroLogicalType" in field.config and "avroType" in field.config:
             return {"type": field.config["avroType"], "logicalType": field.config["avroLogicalType"]}
         if "avroLogicalType" in field.config:
-            if field.config["avroLogicalType"] in ["timestamp-millis", "timestamp-micros", "local-timestamp-millis", "local-timestamp-micros", "time-micros"]:
+            if field.config["avroLogicalType"] in [
+                "timestamp-millis",
+                "timestamp-micros",
+                "local-timestamp-millis",
+                "local-timestamp-micros",
+                "time-micros",
+            ]:
                 return {"type": "long", "logicalType": field.config["avroLogicalType"]}
             if field.config["avroLogicalType"] in ["time-millis", "date"]:
                 return {"type": "int", "logicalType": field.config["avroLogicalType"]}
@@ -82,8 +95,7 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
     elif field.type in ["binary"]:
         return "bytes"
     elif field.type in ["array"]:
-        # TODO support array structs
-        return "array"
+        return {"type": "array", "items": to_avro_type(field.items, field_name)}
     elif field.type in ["null"]:
         return "null"
     else:

datacontract/export/avro_idl_converter.py

@@ -7,28 +7,7 @@ from datacontract.lint.resolve import inline_definitions_into_data_contract
 from datacontract.model.data_contract_specification import DataContractSpecification, Field
 from datacontract.model.exceptions import DataContractException
 
-
-def to_avro_idl(contract: DataContractSpecification) -> str:
-    """Serialize the provided data contract specification into an Avro IDL string.
-
-    The data contract will be serialized as a protocol, with one record type
-    for each contained model. Model fields are mapped one-to-one to Avro IDL
-    record fields.
-    """
-    stream = StringIO()
-    to_avro_idl_stream(contract, stream)
-    return stream.getvalue()
-
-
-def to_avro_idl_stream(contract: DataContractSpecification, stream: typing.TextIO):
-    """Serialize the provided data contract specification into Avro IDL."""
-    ir = _contract_to_avro_idl_ir(contract)
-    if ir.description:
-        stream.write(f"/** {contract.info.description} */\n")
-    stream.write(f"protocol {ir.name or 'Unnamed'} {{\n")
-    for model_type in ir.model_types:
-        _write_model_type(model_type, stream)
-    stream.write("}\n")
+from datacontract.export.exporter import Exporter
 
 
 class AvroPrimitiveType(Enum):
@@ -107,6 +86,34 @@ avro_primitive_types = set(
 )
 
 
+class AvroIdlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_avro_idl(data_contract)
+
+
+def to_avro_idl(contract: DataContractSpecification) -> str:
+    """Serialize the provided data contract specification into an Avro IDL string.
+
+    The data contract will be serialized as a protocol, with one record type
+    for each contained model. Model fields are mapped one-to-one to Avro IDL
+    record fields.
+    """
+    stream = StringIO()
+    to_avro_idl_stream(contract, stream)
+    return stream.getvalue()
+
+
+def to_avro_idl_stream(contract: DataContractSpecification, stream: typing.TextIO):
+    """Serialize the provided data contract specification into Avro IDL."""
+    ir = _contract_to_avro_idl_ir(contract)
+    if ir.description:
+        stream.write(f"/** {contract.info.description} */\n")
+    stream.write(f"protocol {ir.name or 'Unnamed'} {{\n")
+    for model_type in ir.model_types:
+        _write_model_type(model_type, stream)
+    stream.write("}\n")
+
+
 def _to_avro_primitive_logical_type(field_name: str, field: Field) -> AvroPrimitiveField:
     result = AvroPrimitiveField(field_name, field.required, field.description, AvroPrimitiveType.string)
     match field.type:

datacontract/export/bigquery_converter.py

@@ -5,6 +5,21 @@ from typing import Dict, List
 from datacontract.model.data_contract_specification import Model, Field, Server
 from datacontract.model.exceptions import DataContractException
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class BigQueryExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        self.dict_args = export_args
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        found_server = data_contract.servers.get(server)
+        if found_server is None:
+            raise RuntimeError("Export to bigquery requires selecting a bigquery server from the data contract.")
+        if found_server.type != "bigquery":
+            raise RuntimeError("Export to bigquery requires selecting a bigquery server from the data contract.")
+
+        return to_bigquery_json(model_name, model_value, found_server)
+
 
 def to_bigquery_json(model_name: str, model_value: Model, server: Server) -> str:
     bigquery_table = to_bigquery_schema(model_name, model_value, server)

datacontract/export/dbml_converter.py

@@ -8,6 +8,15 @@ import datacontract.model.data_contract_specification as spec
 from datacontract.export.sql_type_converter import convert_to_sql_type
 
 
+from datacontract.export.exporter import Exporter
+
+
+class DbmlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        found_server = data_contract.servers.get(server)
+        return to_dbml_diagram(data_contract, found_server)
+
+
 def to_dbml_diagram(contract: spec.DataContractSpecification, server: spec.Server) -> str:
     result = ""
     result += add_generated_info(contract, server) + "\n"

datacontract/export/dbt_converter.py

@@ -5,6 +5,28 @@ import yaml
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class DbtExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_dbt_models_yaml(data_contract)
+
+
+class DbtSourceExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_dbt_sources_yaml(data_contract, server)
+
+
+class DbtStageExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_dbt_staging_sql(
+            data_contract,
+            model_name,
+            model_value,
+        )
+
 
 def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
     dbt = {
@@ -19,7 +41,10 @@ def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
 
 def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name: str, model_value: Model) -> str:
     if data_contract_spec.models is None or len(data_contract_spec.models.items()) != 1:
-        print("Export to dbt-staging-sql currently only works with exactly one model in the data contract.")
+        print(
+            "Export to dbt-staging-sql currently only works with exactly one model in the data contract."
+            "Please specify the model name."
+        )
         return ""
 
     id = data_contract_spec.id

datacontract/export/exporter.py

@@ -0,0 +1,88 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+import typing
+
+from datacontract.model.data_contract_specification import DataContractSpecification
+
+
+class Exporter(ABC):
+    def __init__(self, export_format) -> None:
+        self.export_format = export_format
+
+    @abstractmethod
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        pass
+
+
+class ExportFormat(str, Enum):
+    jsonschema = "jsonschema"
+    pydantic_model = "pydantic-model"
+    sodacl = "sodacl"
+    dbt = "dbt"
+    dbt_sources = "dbt-sources"
+    dbt_staging_sql = "dbt-staging-sql"
+    odcs = "odcs"
+    rdf = "rdf"
+    avro = "avro"
+    protobuf = "protobuf"
+    great_expectations = "great-expectations"
+    terraform = "terraform"
+    avro_idl = "avro-idl"
+    sql = "sql"
+    sql_query = "sql-query"
+    html = "html"
+    go = "go"
+    bigquery = "bigquery"
+    dbml = "dbml"
+    spark = "spark"
+
+    @classmethod
+    def get_suported_formats(cls):
+        return list(map(lambda c: c.value, cls))
+
+
+def _check_models_for_export(
+    data_contract: DataContractSpecification, model: str, export_format: str
+) -> typing.Tuple[str, str]:
+    if data_contract.models is None:
+        raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
+
+    model_names = list(data_contract.models.keys())
+
+    if model == "all":
+        if len(data_contract.models.items()) != 1:
+            raise RuntimeError(
+                f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
+            )
+
+        model_name, model_value = next(iter(data_contract.models.items()))
+    else:
+        model_name = model
+        model_value = data_contract.models.get(model_name)
+        if model_value is None:
+            raise RuntimeError(f"Model {model_name} not found in the data contract. Available models: {model_names}")
+
+    return model_name, model_value
+
+
+def _determine_sql_server_type(data_contract: DataContractSpecification, sql_server_type: str, server: str = None):
+    if sql_server_type == "auto":
+        if data_contract.servers is None or len(data_contract.servers) == 0:
+            raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
+
+        if server is None:
+            server_types = set([server.type for server in data_contract.servers.values()])
+        else:
+            server_types = {data_contract.servers[server].type}
+
+        if "snowflake" in server_types:
+            return "snowflake"
+        elif "postgres" in server_types:
+            return "postgres"
+        elif "databricks" in server_types:
+            return "databricks"
+        else:
+            # default to snowflake dialect
+            return "snowflake"
+    else:
+        return sql_server_type

datacontract/export/exporter_factory.py

@@ -0,0 +1,145 @@
+import importlib
+import sys
+from datacontract.export.exporter import ExportFormat, Exporter
+
+
+class ExporterFactory:
+    def __init__(self):
+        self.dict_exporter = {}
+        self.dict_lazy_exporter = {}
+
+    def register_exporter(self, name: str, exporter: Exporter):
+        self.dict_exporter.update({name: exporter})
+
+    def register_lazy_exporter(self, name: str, module_path: str, class_name: str):
+        self.dict_lazy_exporter.update({name: (module_path, class_name)})
+
+    def create(self, name) -> Exporter:
+        exporters = self.dict_exporter.copy()
+        exporters.update(self.dict_lazy_exporter.copy())
+        if name not in exporters.keys():
+            raise ValueError(f"The '{name}' format is not supported.")
+        exporter_class = exporters[name]
+        if type(exporters[name]) is tuple:
+            exporter_class = load_module_class(module_path=exporters[name][0], class_name=exporters[name][1])
+        if not exporter_class:
+            raise ValueError(f"Module {name} could not be loaded.")
+        return exporter_class(name)
+
+
+def import_module(module_path):
+    if importlib.util.find_spec(module_path) is not None:
+        try:
+            module = importlib.import_module(module_path)
+        except ModuleNotFoundError:
+            return None
+        sys.modules[module_path] = module
+        return module
+
+
+def load_module_class(module_path, class_name):
+    module = import_module(module_path)
+    if not module:
+        return None
+    return getattr(module, class_name)
+
+
+exporter_factory = ExporterFactory()
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.avro, module_path="datacontract.export.avro_converter", class_name="AvroExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.avro_idl,
+    module_path="datacontract.export.avro_idl_converter",
+    class_name="AvroIdlExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.bigquery,
+    module_path="datacontract.export.bigquery_converter",
+    class_name="BigQueryExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbml, module_path="datacontract.export.dbml_converter", class_name="DbmlExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.rdf, module_path="datacontract.export.rdf_converter", class_name="RdfExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbt, module_path="datacontract.export.dbt_converter", class_name="DbtExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbt_sources,
+    module_path="datacontract.export.dbt_converter",
+    class_name="DbtSourceExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbt_staging_sql,
+    module_path="datacontract.export.dbt_converter",
+    class_name="DbtStageExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.jsonschema,
+    module_path="datacontract.export.jsonschema_converter",
+    class_name="JsonSchemaExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.odcs, module_path="datacontract.export.odcs_converter", class_name="OdcsExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.go, module_path="datacontract.export.go_converter", class_name="GoExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.great_expectations,
+    module_path="datacontract.export.great_expectations_converter",
+    class_name="GreateExpectationsExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.html, module_path="datacontract.export.html_export", class_name="HtmlExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.protobuf,
+    module_path="datacontract.export.protobuf_converter",
+    class_name="ProtoBufExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.pydantic_model,
+    module_path="datacontract.export.pydantic_converter",
+    class_name="PydanticExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sodacl, module_path="datacontract.export.sodacl_converter", class_name="SodaExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sql, module_path="datacontract.export.sql_converter", class_name="SqlExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sql_query, module_path="datacontract.export.sql_converter", class_name="SqlQueryExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.terraform,
+    module_path="datacontract.export.terraform_converter",
+    class_name="TerraformExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.spark, module_path="datacontract.export.spark_converter", class_name="SparkExporter"
+)

datacontract/export/go_converter.py

@@ -1,6 +1,12 @@
 import datacontract.model.data_contract_specification as spec
 from typing import List
 import re
+from datacontract.export.exporter import Exporter
+
+
+class GoExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_go_types(data_contract)
 
 
 def to_go_types(contract: spec.DataContractSpecification) -> str:

datacontract/export/great_expectations_converter.py

@@ -4,6 +4,16 @@ from typing import Dict, List, Any
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Quality
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class GreateExpectationsExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_great_expectations(
+            data_contract,
+            model_name,
+        )
 
 
 def to_great_expectations(data_contract_spec: DataContractSpecification, model_key: str) -> str:

datacontract/export/html_export.py

@@ -8,6 +8,12 @@ import yaml
 from jinja2 import Environment, PackageLoader, select_autoescape
 
 from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.export.exporter import Exporter
+
+
+class HtmlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_html(data_contract)
 
 
 def to_html(data_contract_spec: DataContractSpecification) -> str:

datacontract/export/jsonschema_converter.py

@@ -3,6 +3,14 @@ from typing import Dict
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class JsonSchemaExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_jsonschema_json(model_name, model_value)
+
 
 def to_jsonschemas(data_contract_spec: DataContractSpecification):
     jsonschmemas = {}
@@ -17,21 +25,6 @@ def to_jsonschema_json(model_key, model_value: Model) -> str:
     return json.dumps(jsonschema, indent=2)
 
 
-def to_jsonschema(model_key, model_value: Model) -> dict:
-    model = {
-        "$schema": "http://json-schema.org/draft-07/schema#",
-        "type": "object",
-        "properties": to_properties(model_value.fields),
-        "required": to_required(model_value.fields),
-    }
-    if model_value.title:
-        model["title"] = model_value.title
-    if model_value.description:
-        model["description"] = model_value.description
-
-    return model
-
-
 def to_properties(fields: Dict[str, Field]) -> dict:
     properties = {}
     for field_name, field in fields.items():
@@ -65,27 +58,27 @@ def to_property(field: Field) -> dict:
         property["pattern"] = field.pattern
     if field.enum:
         property["enum"] = field.enum
-    if field.minLength:
+    if field.minLength is not None:
         property["minLength"] = field.minLength
-    if field.maxLength:
+    if field.maxLength is not None:
         property["maxLength"] = field.maxLength
     if field.title:
         property["title"] = field.title
     if field.description:
         property["description"] = field.description
-    if field.exclusiveMinimum:
+    if field.exclusiveMinimum is not None:
         property["exclusiveMinimum"] = field.exclusiveMinimum
-    if field.exclusiveMaximum:
+    if field.exclusiveMaximum is not None:
         property["exclusiveMaximum"] = field.exclusiveMaximum
-    if field.minimum:
+    if field.minimum is not None:
         property["minimum"] = field.minimum
-    if field.maximum:
+    if field.maximum is not None:
         property["maximum"] = field.maximum
     if field.tags:
         property["tags"] = field.tags
     if field.pii:
         property["pii"] = field.pii
-    if field.classification:
+    if field.classification is not None:
         property["classification"] = field.classification
 
     # TODO: all constraints
@@ -126,7 +119,7 @@ def convert_type_format(type, format) -> (str, str):
     return None, None
 
 
-def convert_format(format):
+def convert_format(self, format):
     if format is None:
         return None
     if format.lower() in ["uri"]:
@@ -138,3 +131,18 @@ def convert_format(format):
     if format.lower() in ["boolean"]:
         return "boolean"
     return None
+
+
+def to_jsonschema(model_key, model_value: Model) -> dict:
+    model = {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "type": "object",
+        "properties": to_properties(model_value.fields),
+        "required": to_required(model_value.fields),
+    }
+    if model_value.title:
+        model["title"] = model_value.title
+    if model_value.description:
+        model["description"] = model_value.description
+
+    return model

datacontract/export/odcs_converter.py

@@ -3,6 +3,12 @@ from typing import Dict
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.export.exporter import Exporter
+
+
+class OdcsExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_odcs_yaml(data_contract)
 
 
 def to_odcs_yaml(data_contract_spec: DataContractSpecification):
@@ -24,13 +30,30 @@ def to_odcs_yaml(data_contract_spec: DataContractSpecification):
 
     if data_contract_spec.terms is not None:
         odcs["description"] = {
-            "purpose": None,
+            "purpose": data_contract_spec.terms.description.strip()
+            if data_contract_spec.terms.description is not None
+            else None,
             "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
             "limitations": data_contract_spec.terms.limitations.strip()
             if data_contract_spec.terms.limitations is not None
             else None,
         }
 
+    if data_contract_spec.servicelevels is not None:
+        slas = []
+        if data_contract_spec.servicelevels.availability is not None:
+            slas.append(
+                {
+                    "property": "generalAvailability",
+                    "value": data_contract_spec.servicelevels.availability.description,
+                }
+            )
+        if data_contract_spec.servicelevels.retention is not None:
+            slas.append({"property": "retention", "value": data_contract_spec.servicelevels.retention.period})
+
+        if len(slas) > 0:
+            odcs["slaProperties"] = slas
+
     odcs["type"] = "tables"  # required, TODO read from models.type?
     odcs["dataset"] = []
 

datacontract/export/protobuf_converter.py

@@ -1,4 +1,10 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.export.exporter import Exporter
+
+
+class ProtoBufExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_protobuf(data_contract)
 
 
 def to_protobuf(data_contract_spec: DataContractSpecification):

datacontract/export/pydantic_converter.py

@@ -2,6 +2,12 @@ import ast
 import typing
 
 import datacontract.model.data_contract_specification as spec
+from datacontract.export.exporter import Exporter
+
+
+class PydanticExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_pydantic_model_str(data_contract)
 
 
 def to_pydantic_model_str(contract: spec.DataContractSpecification) -> str:

datacontract/export/rdf_converter.py

@@ -3,6 +3,15 @@ from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
 
 from datacontract.model.data_contract_specification import DataContractSpecification
 
+from datacontract.export.exporter import Exporter
+
+
+class RdfExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        self.dict_args = export_args
+        rdf_base = self.dict_args.get("rdf_base")
+        return to_rdf_n3(data_contract_spec=data_contract, base=rdf_base)
+
 
 def is_literal(property_name):
     return property_name in [