datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/export/bigquery_converter.py

@@ -0,0 +1,130 @@
+import json
+import logging
+from typing import Dict, List
+
+from datacontract.export.exporter import Exporter, _check_models_for_export
+from datacontract.model.data_contract_specification import Field, Model, Server
+from datacontract.model.exceptions import DataContractException
+
+
+class BigQueryExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        self.dict_args = export_args
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        found_server = data_contract.servers.get(server)
+        if found_server is None:
+            raise RuntimeError("Export to bigquery requires selecting a bigquery server from the data contract.")
+        if found_server.type != "bigquery":
+            raise RuntimeError("Export to bigquery requires selecting a bigquery server from the data contract.")
+
+        return to_bigquery_json(model_name, model_value, found_server)
+
+
+def to_bigquery_json(model_name: str, model_value: Model, server: Server) -> str:
+    bigquery_table = to_bigquery_schema(model_name, model_value, server)
+    return json.dumps(bigquery_table, indent=2)
+
+
+def to_bigquery_schema(model_name: str, model_value: Model, server: Server) -> dict:
+    return {
+        "kind": "bigquery#table",
+        "tableReference": {"datasetId": server.dataset, "projectId": server.project, "tableId": model_name},
+        "description": model_value.description,
+        "schema": {"fields": to_fields_array(model_value.fields)},
+    }
+
+
+def to_fields_array(fields: Dict[str, Field]) -> List[Dict[str, Field]]:
+    bq_fields = []
+    for field_name, field in fields.items():
+        bq_fields.append(to_field(field_name, field))
+
+    return bq_fields
+
+
+def to_field(field_name: str, field: Field) -> dict:
+    bq_type = map_type_to_bigquery(field)
+    bq_field = {
+        "name": field_name,
+        "type": bq_type,
+        "mode": "REQUIRED" if field.required else "NULLABLE",
+        "description": field.description,
+    }
+
+    # handle arrays
+    if field.type == "array":
+        bq_field["mode"] = "REPEATED"
+        if field.items.type == "object":
+            # in case the array type is a complex object, we want to copy all its fields
+            bq_field["fields"] = to_fields_array(field.items.fields)
+        else:
+            bq_field["type"] = map_type_to_bigquery(field.items)
+
+    # all of these can carry other fields
+    elif bq_type.lower() in ["record", "struct"]:
+        bq_field["fields"] = to_fields_array(field.fields)
+
+    # strings can have a maxlength
+    if bq_type.lower() == "string":
+        bq_field["maxLength"] = field.maxLength
+
+    # number types have precision and scale
+    if bq_type.lower() in ["numeric", "bignumeric"]:
+        bq_field["precision"] = field.precision
+        bq_field["scale"] = field.scale
+
+    return bq_field
+
+
+def map_type_to_bigquery(field: Field) -> str:
+    logger = logging.getLogger(__name__)
+
+    field_type = field.type
+    if not field_type:
+        return None
+
+    if field.config and "bigqueryType" in field.config:
+        return field.config["bigqueryType"]
+
+    if field_type.lower() in ["string", "varchar", "text"]:
+        return "STRING"
+    elif field_type.lower() == "bytes":
+        return "BYTES"
+    elif field_type.lower() in ["int", "integer"]:
+        return "INTEGER"
+    elif field_type.lower() in ["long", "bigint"]:
+        return "INT64"
+    elif field_type.lower() == "float":
+        return "FLOAT64"
+    elif field_type.lower() == "boolean":
+        return "BOOL"
+    elif field_type.lower() in ["timestamp", "timestamp_tz"]:
+        return "TIMESTAMP"
+    elif field_type.lower() == "date":
+        return "DATE"
+    elif field_type.lower() == "timestamp_ntz":
+        return "DATETIME"
+    elif field_type.lower() in ["number", "decimal", "numeric"]:
+        return "NUMERIC"
+    elif field_type.lower() == "double":
+        return "BIGNUMERIC"
+    elif field_type.lower() in ["object", "record"] and not field.fields:
+        return "JSON"
+    elif field_type.lower() in ["object", "record", "array"]:
+        return "RECORD"
+    elif field_type.lower() == "struct":
+        return "STRUCT"
+    elif field_type.lower() == "null":
+        logger.info(
+            f"Can't properly map {field.title} to bigquery Schema, as 'null' \
+                 is not supported as a type. Mapping it to STRING."
+        )
+        return "STRING"
+    else:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="Map datacontract type to bigquery data type",
+            reason=f"Unsupported type {field_type} in data contract definition.",
+            engine="datacontract",
+        )

datacontract/export/custom_converter.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Model,
+)
+
+
+class CustomExporter(Exporter):
+    """Exporter implementation for converting data contracts to Markdown."""
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model: Model,
+        server: str,
+        sql_server_type: str,
+        export_args: dict,
+    ) -> str:
+        """Exports a data contract to custom format with Jinja."""
+        template = export_args.get("template")
+        if template is None:
+            raise RuntimeError("Export to custom requires template argument.")
+
+        return to_custom(data_contract, template)
+
+
+def to_custom(data_contract: DataContractSpecification, template_path: Path) -> str:
+    template = get_template(template_path)
+    rendered_sql = template.render(data_contract=data_contract)
+    return rendered_sql
+
+
+def get_template(path: Path):
+    abosolute_path = Path(path).resolve()
+    env = Environment(loader=FileSystemLoader(str(abosolute_path.parent)))
+    return env.get_template(path.name)

datacontract/export/data_caterer_converter.py

@@ -0,0 +1,161 @@
+from typing import Dict
+
+import yaml
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server
+
+
+class DataCatererExporter(Exporter):
+    """
+    Exporter class for Data Caterer.
+    Creates a YAML file, based on the data contract, for Data Caterer to generate synthetic data.
+    """
+
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_data_caterer_generate_yaml(data_contract, server)
+
+
+def to_data_caterer_generate_yaml(data_contract_spec: DataContractSpecification, server):
+    generation_task = {"name": data_contract_spec.info.title, "steps": []}
+    server_info = _get_server_info(data_contract_spec, server)
+
+    for model_key, model_value in data_contract_spec.models.items():
+        odcs_table = _to_data_caterer_generate_step(model_key, model_value, server_info)
+        generation_task["steps"].append(odcs_table)
+    return yaml.dump(generation_task, indent=2, sort_keys=False, allow_unicode=True)
+
+
+def _get_server_info(data_contract_spec: DataContractSpecification, server):
+    if server is not None and server in data_contract_spec.servers:
+        return data_contract_spec.servers.get(server)
+    elif server is not None:
+        raise Exception(f"Server name not found in servers list in data contract, server-name={server}")
+    elif len(data_contract_spec.servers.keys()) > 0:
+        return next(iter(data_contract_spec.servers.values()))
+    else:
+        return None
+
+
+def _to_data_caterer_generate_step(model_key, model_value: Model, server: Server) -> dict:
+    step = {
+        "name": model_key,
+        "type": _to_step_type(server),
+        "options": _to_data_source_options(model_key, server),
+        "fields": [],
+    }
+    fields = _to_fields(model_value.fields)
+    if fields:
+        step["fields"] = fields
+    return step
+
+
+def _to_step_type(server: Server):
+    if server is not None and server.type is not None:
+        if server.type in ["s3", "gcs", "azure", "local"]:
+            return server.format
+        else:
+            return server.type
+    else:
+        return "csv"
+
+
+def _to_data_source_options(model_key, server: Server):
+    options = {}
+    if server is not None and server.type is not None:
+        if server.type in ["s3", "gcs", "azure", "local"]:
+            if server.path is not None:
+                options["path"] = server.path
+            elif server.location is not None:
+                options["path"] = server.location
+            else:
+                options["path"] = "/tmp/data_caterer_data"
+        elif server.type == "postgres":
+            options["schema"] = server.schema_
+            options["table"] = model_key
+        elif server.type == "kafka":
+            options["topic"] = server.topic
+
+    return options
+
+
+def _to_fields(fields: Dict[str, Field]) -> list:
+    dc_fields = []
+    for field_name, field in fields.items():
+        column = _to_field(field_name, field)
+        dc_fields.append(column)
+    return dc_fields
+
+
+def _to_field(field_name: str, field: Field) -> dict:
+    dc_field = {"name": field_name}
+    dc_generator_opts = {}
+
+    if field.type is not None:
+        new_type = _to_data_type(field.type)
+        dc_field["type"] = _to_data_type(field.type)
+        if new_type == "object" or new_type == "record" or new_type == "struct":
+            # need to get nested field definitions
+            nested_fields = _to_fields(field.fields)
+            dc_field["fields"] = nested_fields
+        elif new_type == "array":
+            if field.items is not None and field.items.type is not None:
+                dc_generator_opts["arrayType"] = _to_data_type(field.items.type)
+            else:
+                dc_generator_opts["arrayType"] = "string"
+
+    if field.enum is not None and len(field.enum) > 0:
+        dc_generator_opts["oneOf"] = field.enum
+    if field.unique is not None and field.unique:
+        dc_generator_opts["isUnique"] = field.unique
+    if field.primaryKey is not None and field.primaryKey:
+        dc_generator_opts["isPrimaryKey"] = field.primaryKey
+    if field.minLength is not None:
+        if field.type is not None and field.type == "array":
+            dc_generator_opts["arrayMinLen"] = field.minLength
+        else:
+            dc_generator_opts["minLen"] = field.minLength
+    if field.maxLength is not None:
+        if field.type is not None and field.type == "array":
+            dc_generator_opts["arrayMaxLen"] = field.maxLength
+        else:
+            dc_generator_opts["maxLen"] = field.maxLength
+    if field.pattern is not None:
+        dc_generator_opts["regex"] = field.pattern
+    if field.minimum is not None:
+        dc_generator_opts["min"] = field.minimum
+    if field.maximum is not None:
+        dc_generator_opts["max"] = field.maximum
+
+    if len(dc_generator_opts.keys()) > 0:
+        dc_field["options"] = dc_generator_opts
+    return dc_field
+
+
+def _to_data_type(data_type):
+    if data_type == "number" or data_type == "numeric" or data_type == "double":
+        return "double"
+    elif data_type == "decimal" or data_type == "bigint":
+        return "decimal"
+    elif data_type == "int" or data_type == "integer":
+        return "integer"
+    elif data_type == "long":
+        return "long"
+    elif data_type == "float":
+        return "float"
+    elif data_type == "string" or data_type == "text" or data_type == "varchar":
+        return "string"
+    if data_type == "boolean":
+        return "boolean"
+    if data_type == "timestamp" or data_type == "timestamp_tz" or data_type == "timestamp_ntz":
+        return "timestamp"
+    elif data_type == "date":
+        return "date"
+    elif data_type == "array":
+        return "array"
+    elif data_type == "map" or data_type == "object" or data_type == "record" or data_type == "struct":
+        return "struct"
+    elif data_type == "bytes":
+        return "binary"
+    else:
+        return "string"

datacontract/export/dbml_converter.py

@@ -0,0 +1,148 @@
+from datetime import datetime
+from importlib.metadata import version
+from typing import Tuple
+
+import pytz
+
+import datacontract.model.data_contract_specification as spec
+from datacontract.export.exporter import Exporter
+from datacontract.export.sql_type_converter import convert_to_sql_type
+from datacontract.model.exceptions import DataContractException
+
+
+class DbmlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        found_server = data_contract.servers.get(server)
+        return to_dbml_diagram(data_contract, found_server)
+
+
+def to_dbml_diagram(contract: spec.DataContractSpecification, server: spec.Server) -> str:
+    result = ""
+    result += add_generated_info(contract, server) + "\n"
+    result += generate_project_info(contract) + "\n"
+
+    for model_name, model in contract.models.items():
+        table_description = generate_table(model_name, model, server)
+        result += f"\n{table_description}\n"
+
+    return result
+
+
+def add_generated_info(contract: spec.DataContractSpecification, server: spec.Server) -> str:
+    tz = pytz.timezone("UTC")
+    now = datetime.now(tz)
+    formatted_date = now.strftime("%b %d %Y")
+    datacontract_cli_version = get_version()
+    dialect = "Logical Datacontract" if server is None else server.type
+
+    generated_info = """
+Generated at {0} by datacontract-cli version {1}
+for datacontract {2} ({3}) version {4} 
+Using {5} Types for the field types
+    """.format(
+        formatted_date, datacontract_cli_version, contract.info.title, contract.id, contract.info.version, dialect
+    )
+
+    comment = """/*
+{0}
+*/
+    """.format(generated_info)
+    return comment
+
+
+def get_version() -> str:
+    try:
+        return version("datacontract_cli")
+    except Exception:
+        return ""
+
+
+def generate_project_info(contract: spec.DataContractSpecification) -> str:
+    return """Project "{0}" {{
+    Note: '''{1}'''
+}}\n
+    """.format(contract.info.title, contract.info.description)
+
+
+def generate_table(model_name: str, model: spec.Model, server: spec.Server) -> str:
+    result = """Table "{0}" {{ 
+Note: {1}
+    """.format(model_name, formatDescription(model.description))
+
+    references = []
+
+    for field_name, field in model.fields.items():
+        ref, field_string = generate_field(field_name, field, model_name, server)
+        if ref is not None:
+            references.append(ref)
+        result += "{0}\n".format(field_string)
+
+    result += "}\n"
+
+    # and if any: add the references
+    if len(references) > 0:
+        for ref in references:
+            result += "Ref: {0}\n".format(ref)
+
+        result += "\n"
+
+    return result
+
+
+def generate_field(field_name: str, field: spec.Field, model_name: str, server: spec.Server) -> Tuple[str, str]:
+    if field.primaryKey or field.primary:
+        if field.required is not None:
+            if not field.required:
+                raise DataContractException(
+                    type="lint",
+                    name="Primary key fields cannot have required == False.",
+                    result="error",
+                    reason="Primary key fields cannot have required == False.",
+                    engine="datacontract",
+                )
+        else:
+            field.required = True
+        if field.unique is not None:
+            if not field.unique:
+                raise DataContractException(
+                    type="lint",
+                    name="Primary key fields cannot have unique == False",
+                    result="error",
+                    reason="Primary key fields cannot have unique == False.",
+                    engine="datacontract",
+                )
+        else:
+            field.unique = True
+
+    field_attrs = []
+    if field.primaryKey or field.primary:
+        field_attrs.append("pk")
+
+    if field.unique:
+        field_attrs.append("unique")
+
+    if field.required:
+        field_attrs.append("not null")
+    else:
+        field_attrs.append("null")
+
+    if field.description:
+        field_attrs.append("""Note: {0}""".format(formatDescription(field.description)))
+
+    field_type = field.type if server is None else convert_to_sql_type(field, server.type)
+
+    field_str = '"{0}" "{1}" [{2}]'.format(field_name, field_type, ",".join(field_attrs))
+    ref_str = None
+    if (field.references) is not None:
+        if field.unique:
+            ref_str = "{0}.{1} - {2}".format(model_name, field_name, field.references)
+        else:
+            ref_str = "{0}.{1} > {2}".format(model_name, field_name, field.references)
+    return (ref_str, field_str)
+
+
+def formatDescription(input: str) -> str:
+    if "\n" in input or "\r" in input or '"' in input:
+        return "'''{0}'''".format(input)
+    else:
+        return '"{0}"'.format(input)