datacontract-cli 0.10.6__py3-none-any.whl → 0.10.8__py3-none-any.whl

datacontract/export/dbml_converter.py

@@ -8,6 +8,15 @@ import datacontract.model.data_contract_specification as spec
 from datacontract.export.sql_type_converter import convert_to_sql_type
 
 
+from datacontract.export.exporter import Exporter
+
+
+class DbmlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        found_server = data_contract.servers.get(server)
+        return to_dbml_diagram(data_contract, found_server)
+
+
 def to_dbml_diagram(contract: spec.DataContractSpecification, server: spec.Server) -> str:
     result = ""
     result += add_generated_info(contract, server) + "\n"

datacontract/export/dbt_converter.py

@@ -5,6 +5,28 @@ import yaml
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class DbtExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_dbt_models_yaml(data_contract)
+
+
+class DbtSourceExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_dbt_sources_yaml(data_contract, server)
+
+
+class DbtStageExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_dbt_staging_sql(
+            data_contract,
+            model_name,
+            model_value,
+        )
+
 
 def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
     dbt = {
@@ -19,7 +41,10 @@ def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
 
 def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name: str, model_value: Model) -> str:
     if data_contract_spec.models is None or len(data_contract_spec.models.items()) != 1:
-        print("Export to dbt-staging-sql currently only works with exactly one model in the data contract.")
+        print(
+            "Export to dbt-staging-sql currently only works with exactly one model in the data contract."
+            "Please specify the model name."
+        )
         return ""
 
     id = data_contract_spec.id

datacontract/export/exporter.py

@@ -0,0 +1,87 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+import typing
+
+from datacontract.model.data_contract_specification import DataContractSpecification
+
+
+class Exporter(ABC):
+    def __init__(self, export_format) -> None:
+        self.export_format = export_format
+
+    @abstractmethod
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        pass
+
+
+class ExportFormat(str, Enum):
+    jsonschema = "jsonschema"
+    pydantic_model = "pydantic-model"
+    sodacl = "sodacl"
+    dbt = "dbt"
+    dbt_sources = "dbt-sources"
+    dbt_staging_sql = "dbt-staging-sql"
+    odcs = "odcs"
+    rdf = "rdf"
+    avro = "avro"
+    protobuf = "protobuf"
+    great_expectations = "great-expectations"
+    terraform = "terraform"
+    avro_idl = "avro-idl"
+    sql = "sql"
+    sql_query = "sql-query"
+    html = "html"
+    go = "go"
+    bigquery = "bigquery"
+    dbml = "dbml"
+
+    @classmethod
+    def get_formats(cls):
+        return cls.__dict__
+
+
+def _check_models_for_export(
+    data_contract: DataContractSpecification, model: str, export_format: str
+) -> typing.Tuple[str, str]:
+    if data_contract.models is None:
+        raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
+
+    model_names = list(data_contract.models.keys())
+
+    if model == "all":
+        if len(data_contract.models.items()) != 1:
+            raise RuntimeError(
+                f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
+            )
+
+        model_name, model_value = next(iter(data_contract.models.items()))
+    else:
+        model_name = model
+        model_value = data_contract.models.get(model_name)
+        if model_value is None:
+            raise RuntimeError(f"Model {model_name} not found in the data contract. Available models: {model_names}")
+
+    return model_name, model_value
+
+
+def _determine_sql_server_type(data_contract: DataContractSpecification, sql_server_type: str, server: str = None):
+    if sql_server_type == "auto":
+        if data_contract.servers is None or len(data_contract.servers) == 0:
+            raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
+
+        if server is None:
+            server_types = set([server.type for server in data_contract.servers.values()])
+        else:
+            server_types = {data_contract.servers[server].type}
+
+        if "snowflake" in server_types:
+            return "snowflake"
+        elif "postgres" in server_types:
+            return "postgres"
+        elif "databricks" in server_types:
+            return "databricks"
+        else:
+            # default to snowflake dialect
+            return "snowflake"
+    else:
+        return sql_server_type

datacontract/export/exporter_factory.py

@@ -0,0 +1,52 @@
+from datacontract.export.avro_idl_converter import AvroIdlExporter
+from datacontract.export.bigquery_converter import BigQueryExporter
+from datacontract.export.dbml_converter import DbmlExporter
+from datacontract.export.dbt_converter import DbtExporter, DbtSourceExporter, DbtStageExporter
+from datacontract.export.avro_converter import AvroExporter
+from datacontract.export.exporter import ExportFormat, Exporter
+from datacontract.export.go_converter import GoExporter
+from datacontract.export.great_expectations_converter import GreateExpectationsExporter
+from datacontract.export.html_export import HtmlExporter
+from datacontract.export.jsonschema_converter import JsonSchemaExporter
+from datacontract.export.odcs_converter import OdcsExporter
+from datacontract.export.protobuf_converter import ProtoBufExporter
+from datacontract.export.pydantic_converter import PydanticExporter
+from datacontract.export.rdf_converter import RdfExporter
+from datacontract.export.sodacl_converter import SodaExporter
+from datacontract.export.sql_converter import SqlExporter, SqlQueryExporter
+from datacontract.export.terraform_converter import TerraformExporter
+
+
+class ExporterFactory:
+    def __init__(self):
+        self.dict_exporter = {}
+
+    def register_exporter(self, name, exporter):
+        self.dict_exporter.update({name: exporter})
+
+    def create(self, name) -> Exporter:
+        if name not in self.dict_exporter.keys():
+            raise ValueError(f"Export format {name} not supported.")
+        return self.dict_exporter[name](name)
+
+
+exporter_factory = ExporterFactory()
+exporter_factory.register_exporter(ExportFormat.avro, AvroExporter)
+exporter_factory.register_exporter(ExportFormat.avro_idl, AvroIdlExporter)
+exporter_factory.register_exporter(ExportFormat.bigquery, BigQueryExporter)
+exporter_factory.register_exporter(ExportFormat.dbml, DbmlExporter)
+exporter_factory.register_exporter(ExportFormat.rdf, RdfExporter)
+exporter_factory.register_exporter(ExportFormat.dbt, DbtExporter)
+exporter_factory.register_exporter(ExportFormat.dbt_sources, DbtSourceExporter)
+exporter_factory.register_exporter(ExportFormat.dbt_staging_sql, DbtStageExporter)
+exporter_factory.register_exporter(ExportFormat.jsonschema, JsonSchemaExporter)
+exporter_factory.register_exporter(ExportFormat.odcs, OdcsExporter)
+exporter_factory.register_exporter(ExportFormat.go, GoExporter)
+exporter_factory.register_exporter(ExportFormat.great_expectations, GreateExpectationsExporter)
+exporter_factory.register_exporter(ExportFormat.html, HtmlExporter)
+exporter_factory.register_exporter(ExportFormat.protobuf, ProtoBufExporter)
+exporter_factory.register_exporter(ExportFormat.pydantic_model, PydanticExporter)
+exporter_factory.register_exporter(ExportFormat.sodacl, SodaExporter)
+exporter_factory.register_exporter(ExportFormat.sql, SqlExporter)
+exporter_factory.register_exporter(ExportFormat.sql_query, SqlQueryExporter)
+exporter_factory.register_exporter(ExportFormat.terraform, TerraformExporter)

datacontract/export/go_converter.py

@@ -1,6 +1,12 @@
 import datacontract.model.data_contract_specification as spec
 from typing import List
 import re
+from datacontract.export.exporter import Exporter
+
+
+class GoExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_go_types(data_contract)
 
 
 def to_go_types(contract: spec.DataContractSpecification) -> str:

datacontract/export/great_expectations_converter.py

@@ -4,6 +4,16 @@ from typing import Dict, List, Any
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Field, Quality
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class GreateExpectationsExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_great_expectations(
+            data_contract,
+            model_name,
+        )
 
 
 def to_great_expectations(data_contract_spec: DataContractSpecification, model_key: str) -> str:

datacontract/export/html_export.py

@@ -8,6 +8,12 @@ import yaml
 from jinja2 import Environment, PackageLoader, select_autoescape
 
 from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.export.exporter import Exporter
+
+
+class HtmlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_html(data_contract)
 
 
 def to_html(data_contract_spec: DataContractSpecification) -> str:

datacontract/export/jsonschema_converter.py

@@ -3,6 +3,14 @@ from typing import Dict
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class JsonSchemaExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_jsonschema_json(model_name, model_value)
+
 
 def to_jsonschemas(data_contract_spec: DataContractSpecification):
     jsonschmemas = {}
@@ -17,21 +25,6 @@ def to_jsonschema_json(model_key, model_value: Model) -> str:
     return json.dumps(jsonschema, indent=2)
 
 
-def to_jsonschema(model_key, model_value: Model) -> dict:
-    model = {
-        "$schema": "http://json-schema.org/draft-07/schema#",
-        "type": "object",
-        "properties": to_properties(model_value.fields),
-        "required": to_required(model_value.fields),
-    }
-    if model_value.title:
-        model["title"] = model_value.title
-    if model_value.description:
-        model["description"] = model_value.description
-
-    return model
-
-
 def to_properties(fields: Dict[str, Field]) -> dict:
     properties = {}
     for field_name, field in fields.items():
@@ -126,7 +119,7 @@ def convert_type_format(type, format) -> (str, str):
     return None, None
 
 
-def convert_format(format):
+def convert_format(self, format):
     if format is None:
         return None
     if format.lower() in ["uri"]:
@@ -138,3 +131,18 @@ def convert_format(format):
     if format.lower() in ["boolean"]:
         return "boolean"
     return None
+
+
+def to_jsonschema(model_key, model_value: Model) -> dict:
+    model = {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "type": "object",
+        "properties": to_properties(model_value.fields),
+        "required": to_required(model_value.fields),
+    }
+    if model_value.title:
+        model["title"] = model_value.title
+    if model_value.description:
+        model["description"] = model_value.description
+
+    return model

datacontract/export/odcs_converter.py

@@ -3,6 +3,12 @@ from typing import Dict
 import yaml
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.export.exporter import Exporter
+
+
+class OdcsExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_odcs_yaml(data_contract)
 
 
 def to_odcs_yaml(data_contract_spec: DataContractSpecification):
@@ -24,13 +30,30 @@ def to_odcs_yaml(data_contract_spec: DataContractSpecification):
 
     if data_contract_spec.terms is not None:
         odcs["description"] = {
-            "purpose": None,
+            "purpose": data_contract_spec.terms.description.strip()
+            if data_contract_spec.terms.description is not None
+            else None,
             "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
             "limitations": data_contract_spec.terms.limitations.strip()
             if data_contract_spec.terms.limitations is not None
             else None,
         }
 
+    if data_contract_spec.servicelevels is not None:
+        slas = []
+        if data_contract_spec.servicelevels.availability is not None:
+            slas.append(
+                {
+                    "property": "generalAvailability",
+                    "value": data_contract_spec.servicelevels.availability.description,
+                }
+            )
+        if data_contract_spec.servicelevels.retention is not None:
+            slas.append({"property": "retention", "value": data_contract_spec.servicelevels.retention.period})
+
+        if len(slas) > 0:
+            odcs["slaProperties"] = slas
+
     odcs["type"] = "tables"  # required, TODO read from models.type?
     odcs["dataset"] = []
 

datacontract/export/protobuf_converter.py

@@ -1,4 +1,10 @@
 from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.export.exporter import Exporter
+
+
+class ProtoBufExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_protobuf(data_contract)
 
 
 def to_protobuf(data_contract_spec: DataContractSpecification):

datacontract/export/pydantic_converter.py

@@ -2,6 +2,12 @@ import ast
 import typing
 
 import datacontract.model.data_contract_specification as spec
+from datacontract.export.exporter import Exporter
+
+
+class PydanticExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_pydantic_model_str(data_contract)
 
 
 def to_pydantic_model_str(contract: spec.DataContractSpecification) -> str:

datacontract/export/rdf_converter.py

@@ -3,6 +3,15 @@ from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
 
 from datacontract.model.data_contract_specification import DataContractSpecification
 
+from datacontract.export.exporter import Exporter
+
+
+class RdfExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        self.dict_args = export_args
+        rdf_base = self.dict_args.get("rdf_base")
+        return to_rdf_n3(data_contract_spec=data_contract, base=rdf_base)
+
 
 def is_literal(property_name):
     return property_name in [

datacontract/export/sodacl_converter.py

@@ -2,6 +2,12 @@ import yaml
 
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification
+from datacontract.export.exporter import Exporter
+
+
+class SodaExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_sodacl_yaml(data_contract)
 
 
 def to_sodacl_yaml(
@@ -98,7 +104,7 @@ def check_field_min_length(field_name, min_length, quote_field_name: bool = Fals
         field_name = f'"{field_name}"'
     return {
         f"invalid_count({field_name}) = 0": {
-            "name": f"Check that field {field_name} has a min length of {min}",
+            "name": f"Check that field {field_name} has a min length of {min_length}",
             "valid min length": min_length,
         }
     }

datacontract/export/sql_converter.py

@@ -1,6 +1,29 @@
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification, Model
 
+from datacontract.export.exporter import Exporter, _check_models_for_export, _determine_sql_server_type
+
+
+class SqlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        server_type = _determine_sql_server_type(
+            data_contract,
+            sql_server_type,
+        )
+        return to_sql_ddl(data_contract, server_type, export_args.get("server"))
+
+
+class SqlQueryExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        server_type = _determine_sql_server_type(data_contract, sql_server_type, export_args.get("server"))
+        return to_sql_query(
+            data_contract,
+            model_name,
+            model_value,
+            server_type,
+        )
+
 
 def to_sql_query(
     data_contract_spec: DataContractSpecification, model_name: str, model_value: Model, server_type: str = "snowflake"
@@ -37,7 +60,9 @@ def _to_sql_query(model_name, model_value, server_type) -> str:
     return result
 
 
-def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str = "snowflake") -> str:
+def to_sql_ddl(
+    data_contract_spec: DataContractSpecification, server_type: str = "snowflake", server: str = None
+) -> str:
     if data_contract_spec is None:
         return ""
     if data_contract_spec.models is None or len(data_contract_spec.models) == 0:
@@ -45,7 +70,12 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str =
 
     table_prefix = ""
 
-    for server_name, server in iter(data_contract_spec.servers.items()):
+    if server is None:
+        servers = data_contract_spec.servers
+    else:
+        servers = {server: data_contract_spec.servers[server]}
+
+    for server_name, server in iter(servers.items()):
         if server.type == "snowflake":
             server_type = "snowflake"
             break

datacontract/export/sql_type_converter.py

@@ -21,7 +21,7 @@ def convert_to_sql_type(field: Field, server_type: str) -> str:
 # snowflake data types:
 # https://docs.snowflake.com/en/sql-reference/data-types.html
 def convert_to_snowflake(field: Field) -> None | str:
-    if field.config and field.config["snowflakeType"] is not None:
+    if field.config and "snowflakeType" in field.config:
         return field.config["snowflakeType"]
 
     type = field.type
@@ -64,7 +64,7 @@ def convert_to_snowflake(field: Field) -> None | str:
 # https://www.postgresql.org/docs/current/datatype.html
 # Using the name whenever possible
 def convert_type_to_postgres(field: Field) -> None | str:
-    if field.config and field.config["postgresType"] is not None:
+    if field.config and "postgresType" in field.config:
         return field.config["postgresType"]
 
     type = field.type
@@ -109,7 +109,7 @@ def convert_type_to_postgres(field: Field) -> None | str:
 # databricks data types:
 # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
 def convert_to_databricks(field: Field) -> None | str:
-    if field.config and field.config["databricksType"] is not None:
+    if field.config and "databricksType" in field.config:
         return field.config["databricksType"]
     type = field.type
     if type is None:
@@ -161,8 +161,7 @@ def convert_to_duckdb(field: Field) -> None | str:
     if type.lower() in ["time"]:
         return "TIME"  # TIME WITHOUT TIME ZONE
     if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "DECIMAL"
+        return f"DECIMAL({field.precision},{field.scale})"
     if type.lower() in ["float"]:
         return "FLOAT"
     if type.lower() in ["double"]:

datacontract/export/terraform_converter.py

@@ -1,6 +1,12 @@
 import re
 
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
+from datacontract.export.exporter import Exporter
+
+
+class TerraformExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_terraform(data_contract)
 
 
 def to_terraform(data_contract_spec: DataContractSpecification, server_id: str = None) -> str:

datacontract/imports/bigquery_importer.py

@@ -1,8 +1,7 @@
 import json
+import logging
 from typing import List
 
-from google.cloud import bigquery
-
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 from datacontract.model.exceptions import DataContractException
 
@@ -30,6 +29,18 @@ def import_bigquery_from_api(
     bigquery_project: str,
     bigquery_dataset: str,
 ) -> DataContractSpecification:
+    try:
+        from google.cloud import bigquery
+    except ImportError as e:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="bigquery extra missing",
+            reason="Install the extra datacontract-cli[bigquery] to use bigquery",
+            engine="datacontract",
+            original_exception=e,
+        )
+
     client = bigquery.Client(project=bigquery_project)
 
     if bigquery_tables is None:
@@ -63,7 +74,7 @@ def import_bigquery_from_api(
     return data_contract_specification
 
 
-def fetch_table_names(client: bigquery.Client, dataset: str) -> List[str]:
+def fetch_table_names(client, dataset: str) -> List[str]:
     table_names = []
     api_tables = client.list_tables(dataset)
     for api_table in api_tables:
@@ -84,7 +95,9 @@ def convert_bigquery_schema(
     # what exactly leads to friendlyName being set
     table_id = bigquery_schema.get("tableReference").get("tableId")
 
-    data_contract_specification.models[table_id] = Model(fields=fields, type="table")
+    data_contract_specification.models[table_id] = Model(
+        fields=fields, type=map_bigquery_type(bigquery_schema.get("type"))
+    )
 
     # Copy the description, if it exists
     if bigquery_schema.get("description") is not None:
@@ -176,3 +189,16 @@ def map_type_from_bigquery(bigquery_type_str: str):
             reason=f"Unsupported type {bigquery_type_str} in bigquery json definition.",
             engine="datacontract",
         )
+
+
+def map_bigquery_type(bigquery_type: str) -> str:
+    if bigquery_type == "TABLE" or bigquery_type == "EXTERNAL" or bigquery_type == "SNAPSHOT":
+        return "table"
+    elif bigquery_type == "VIEW" or bigquery_type == "MATERIALIZED_VIEW":
+        return "view"
+    else:
+        logger = logging.getLogger(__name__)
+        logger.info(
+            f"Can't properly map bigquery table type '{bigquery_type}' to datacontracts model types. Mapping it to table."
+        )
+        return "table"

datacontract/imports/glue_importer.py

@@ -107,7 +107,7 @@ def get_glue_table_schema(database_name: str, table_name: str):
     return table_schema
 
 
-def import_glue(data_contract_specification: DataContractSpecification, source: str):
+def import_glue(data_contract_specification: DataContractSpecification, source: str, table_names: List[str]):
     """Import the schema of a Glue database."""
 
     catalogid, location_uri = get_glue_database(source)
@@ -116,13 +116,14 @@ def import_glue(data_contract_specification: DataContractSpecification, source:
     if catalogid is None:
         return data_contract_specification
 
-    tables = get_glue_tables(source)
+    if table_names is None:
+        table_names = get_glue_tables(source)
 
     data_contract_specification.servers = {
         "production": Server(type="glue", account=catalogid, database=source, location=location_uri),
     }
 
-    for table_name in tables:
+    for table_name in table_names:
         if data_contract_specification.models is None:
             data_contract_specification.models = {}
 
@@ -141,6 +142,13 @@ def import_glue(data_contract_specification: DataContractSpecification, source:
 
             fields[column["Name"]] = field
 
+            if "decimal" in column["Type"]:
+                # Extract precision and scale from the string
+                perc_scale = column["Type"][8:-1].split(",")
+                print(perc_scale)
+                field.precision = int(perc_scale[0])
+                field.scale = int(perc_scale[1])
+
         data_contract_specification.models[table_name] = Model(
             type="table",
             fields=fields,
@@ -179,5 +187,7 @@ def map_type_from_sql(sql_type: str):
         return "timestamp"
     elif sql_type.lower().startswith("date"):
         return "date"
+    elif sql_type.lower().startswith("decimal"):
+        return "decimal"
     else:
         return "variant"