datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/export/exporter.py

@@ -0,0 +1,100 @@
+import typing
+from abc import ABC, abstractmethod
+from enum import Enum
+
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Model,
+)
+
+
+class Exporter(ABC):
+    def __init__(self, export_format) -> None:
+        self.export_format = export_format
+
+    @abstractmethod
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict | str:
+        pass
+
+
+class ExportFormat(str, Enum):
+    jsonschema = "jsonschema"
+    pydantic_model = "pydantic-model"
+    sodacl = "sodacl"
+    dbt = "dbt"
+    dbt_sources = "dbt-sources"
+    dbt_staging_sql = "dbt-staging-sql"
+    odcs = "odcs"
+    rdf = "rdf"
+    avro = "avro"
+    protobuf = "protobuf"
+    great_expectations = "great-expectations"
+    terraform = "terraform"
+    avro_idl = "avro-idl"
+    sql = "sql"
+    sql_query = "sql-query"
+    mermaid = "mermaid"
+    html = "html"
+    go = "go"
+    bigquery = "bigquery"
+    dbml = "dbml"
+    spark = "spark"
+    sqlalchemy = "sqlalchemy"
+    data_caterer = "data-caterer"
+    dcs = "dcs"
+    markdown = "markdown"
+    iceberg = "iceberg"
+    custom = "custom"
+    excel = "excel"
+    dqx = "dqx"
+
+    @classmethod
+    def get_supported_formats(cls):
+        return list(map(lambda c: c.value, cls))
+
+
+def _check_models_for_export(
+    data_contract: DataContractSpecification, model: str, export_format: str
+) -> typing.Tuple[str, Model]:
+    if data_contract.models is None:
+        raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
+
+    model_names = list(data_contract.models.keys())
+
+    if model == "all":
+        if len(data_contract.models.items()) != 1:
+            raise RuntimeError(
+                f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
+            )
+
+        model_name, model_value = next(iter(data_contract.models.items()))
+    else:
+        model_name = model
+        model_value = data_contract.models.get(model_name)
+        if model_value is None:
+            raise RuntimeError(f"Model {model_name} not found in the data contract. Available models: {model_names}")
+
+    return model_name, model_value
+
+
+def _determine_sql_server_type(data_contract: DataContractSpecification, sql_server_type: str, server: str = None):
+    if sql_server_type == "auto":
+        if data_contract.servers is None or len(data_contract.servers) == 0:
+            raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
+
+        if server is None:
+            server_types = set([server.type for server in data_contract.servers.values()])
+        else:
+            server_types = {data_contract.servers[server].type}
+
+        if "snowflake" in server_types:
+            return "snowflake"
+        elif "postgres" in server_types:
+            return "postgres"
+        elif "databricks" in server_types:
+            return "databricks"
+        else:
+            # default to snowflake dialect
+            return "snowflake"
+    else:
+        return sql_server_type

datacontract/export/exporter_factory.py

@@ -0,0 +1,216 @@
+import importlib
+import sys
+
+from datacontract.export.exporter import Exporter, ExportFormat
+
+
+class ExporterFactory:
+    def __init__(self):
+        self.dict_exporter = {}
+        self.dict_lazy_exporter = {}
+
+    def register_exporter(self, name: str, exporter: Exporter):
+        self.dict_exporter.update({name: exporter})
+
+    def register_lazy_exporter(self, name: str, module_path: str, class_name: str):
+        self.dict_lazy_exporter.update({name: (module_path, class_name)})
+
+    def create(self, name) -> Exporter:
+        exporters = self.dict_exporter.copy()
+        exporters.update(self.dict_lazy_exporter.copy())
+        if name not in exporters.keys():
+            raise ValueError(f"The '{name}' format is not supported.")
+        exporter_class = exporters[name]
+        if type(exporters[name]) is tuple:
+            exporter_class = load_module_class(module_path=exporters[name][0], class_name=exporters[name][1])
+        if not exporter_class:
+            raise ValueError(f"Module {name} could not be loaded.")
+        return exporter_class(name)
+
+
+def import_module(module_path):
+    if importlib.util.find_spec(module_path) is not None:
+        try:
+            module = importlib.import_module(module_path)
+        except ModuleNotFoundError:
+            return None
+        sys.modules[module_path] = module
+        return module
+
+
+def load_module_class(module_path, class_name):
+    module = import_module(module_path)
+    if not module:
+        return None
+    return getattr(module, class_name)
+
+
+exporter_factory = ExporterFactory()
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.avro,
+    module_path="datacontract.export.avro_converter",
+    class_name="AvroExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.avro_idl,
+    module_path="datacontract.export.avro_idl_converter",
+    class_name="AvroIdlExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.bigquery,
+    module_path="datacontract.export.bigquery_converter",
+    class_name="BigQueryExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.data_caterer,
+    module_path="datacontract.export.data_caterer_converter",
+    class_name="DataCatererExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbml,
+    module_path="datacontract.export.dbml_converter",
+    class_name="DbmlExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.rdf,
+    module_path="datacontract.export.rdf_converter",
+    class_name="RdfExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbt,
+    module_path="datacontract.export.dbt_converter",
+    class_name="DbtExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.mermaid,
+    module_path="datacontract.export.mermaid_exporter",
+    class_name="MermaidExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbt_sources,
+    module_path="datacontract.export.dbt_converter",
+    class_name="DbtSourceExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dbt_staging_sql,
+    module_path="datacontract.export.dbt_converter",
+    class_name="DbtStageExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.jsonschema,
+    module_path="datacontract.export.jsonschema_converter",
+    class_name="JsonSchemaExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.odcs,
+    module_path="datacontract.export.odcs_v3_exporter",
+    class_name="OdcsV3Exporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.go,
+    module_path="datacontract.export.go_converter",
+    class_name="GoExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.great_expectations,
+    module_path="datacontract.export.great_expectations_converter",
+    class_name="GreatExpectationsExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.html,
+    module_path="datacontract.export.html_exporter",
+    class_name="HtmlExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.protobuf,
+    module_path="datacontract.export.protobuf_converter",
+    class_name="ProtoBufExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.pydantic_model,
+    module_path="datacontract.export.pydantic_converter",
+    class_name="PydanticExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sodacl,
+    module_path="datacontract.export.sodacl_converter",
+    class_name="SodaExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sql,
+    module_path="datacontract.export.sql_converter",
+    class_name="SqlExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sql_query,
+    module_path="datacontract.export.sql_converter",
+    class_name="SqlQueryExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.terraform,
+    module_path="datacontract.export.terraform_converter",
+    class_name="TerraformExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.spark,
+    module_path="datacontract.export.spark_converter",
+    class_name="SparkExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.sqlalchemy,
+    module_path="datacontract.export.sqlalchemy_converter",
+    class_name="SQLAlchemyExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dcs,
+    module_path="datacontract.export.dcs_exporter",
+    class_name="DcsExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.markdown,
+    module_path="datacontract.export.markdown_converter",
+    class_name="MarkdownExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.dqx,
+    module_path="datacontract.export.dqx_converter",
+    class_name="DqxExporter",
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.custom, module_path="datacontract.export.custom_converter", class_name="CustomExporter"
+)
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.excel, module_path="datacontract.export.excel_exporter", class_name="ExcelExporter"
+)

datacontract/export/go_converter.py

@@ -0,0 +1,105 @@
+import re
+from typing import List
+
+import datacontract.model.data_contract_specification as spec
+from datacontract.export.exporter import Exporter
+
+
+class GoExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_go_types(data_contract)
+
+
+def to_go_types(contract: spec.DataContractSpecification) -> str:
+    result = "package main\n\n"
+
+    for key in contract.models.keys():
+        go_types = generate_go_type(contract.models[key], key)
+        for go_type in go_types:
+            # print(go_type + "\n\n")
+            result += f"\n{go_type}\n"
+
+    return result
+
+
+def python_type_to_go_type(py_type) -> str:
+    match py_type:
+        case "text":
+            return "string"
+        case "timestamp":
+            return "time.Time"
+        case "long":
+            return "int64"
+        case "int":
+            return "int"
+        case "float":
+            return "float64"
+        case "boolean":
+            return "bool"
+        case _:
+            return "interface{}"
+
+
+def to_camel_case(snake_str) -> str:
+    return "".join(word.capitalize() for word in re.split(r"_|(?<!^)(?=[A-Z])", snake_str))
+
+
+def get_subtype(field_info, nested_types, type_name, camel_case_name) -> str:
+    go_type = "interface{}"
+    if field_info.fields:
+        nested_type_name = to_camel_case(f"{type_name}_{camel_case_name}")
+        nested_types[nested_type_name] = field_info.fields
+        go_type = nested_type_name
+
+    match field_info.type:
+        case "array":
+            if field_info.items:
+                item_type = get_subtype(field_info.items, nested_types, type_name, camel_case_name + "Item")
+                go_type = f"[]{item_type}"
+            else:
+                go_type = "[]interface{}"
+        case "record":
+            if field_info.fields:
+                nested_type_name = to_camel_case(f"{type_name}_{camel_case_name}")
+                nested_types[nested_type_name] = field_info.fields
+                go_type = nested_type_name
+            else:
+                go_type = "interface{}"
+        case "object":
+            pass
+        case _:
+            go_type = field_info.type
+
+    return go_type
+
+
+def generate_go_type(model, model_name) -> List[str]:
+    go_types = []
+    type_name = to_camel_case(model_name)
+    lines = [f"type {type_name} struct {{"]
+
+    nested_types = {}
+
+    for field_name, field_info in model.fields.items():
+        go_type = python_type_to_go_type(field_info.type)
+        camel_case_name = to_camel_case(field_name)
+        json_tag = field_name if field_info.required else f"{field_name},omitempty"
+        avro_tag = field_name
+
+        if go_type == "interface{}":
+            go_type = get_subtype(field_info, nested_types, type_name, camel_case_name)
+
+        go_type = go_type if field_info.required else f"*{go_type}"
+
+        lines.append(
+            f'    {camel_case_name} {go_type} `json:"{json_tag}" avro:"{avro_tag}"`  // {field_info.description}'
+        )
+    lines.append("}")
+    go_types.append("\n".join(lines))
+
+    for nested_type_name, nested_fields in nested_types.items():
+        nested_model = spec.Model(fields=nested_fields)
+        nested_go_types = generate_go_type(nested_model, nested_type_name)
+        go_types.extend(nested_go_types)
+
+    return go_types