datacontract-cli 0.10.4__py3-none-any.whl → 0.10.6__py3-none-any.whl

datacontract/breaking/breaking.py

@@ -256,13 +256,13 @@ def field_breaking_changes(
                 )
             )
             continue
-        
-        if field_definition_field == "items" and old_field.type == 'array' and new_field.type == 'array':
+
+        if field_definition_field == "items" and old_field.type == "array" and new_field.type == "array":
             results.extend(
                 field_breaking_changes(
                     old_field=old_value,
                     new_field=new_value,
-                    composition=composition + ['items'],
+                    composition=composition + ["items"],
                     new_path=new_path,
                     include_severities=include_severities,
                 )

datacontract/catalog/catalog.py

@@ -7,8 +7,7 @@ from jinja2 import PackageLoader, Environment, select_autoescape
 
 from datacontract.data_contract import DataContract
 from datacontract.export.html_export import get_version
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 def create_data_contract_html(contracts, file: Path, path: Path):

datacontract/cli.py

@@ -2,6 +2,7 @@ from enum import Enum
 from importlib import metadata
 from pathlib import Path
 from typing import Iterable, Optional
+from typing import List
 
 import typer
 from click import Context
@@ -10,15 +11,12 @@ from rich.console import Console
 from rich.table import Table
 from typer.core import TyperGroup
 from typing_extensions import Annotated
-from typing import List
 
 from datacontract.catalog.catalog import create_index_html, create_data_contract_html
 from datacontract.data_contract import DataContract
 from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
-
 from datacontract.publish.publish import publish_to_datamesh_manager
 
-
 console = Console()
 
 
@@ -162,6 +160,7 @@ class ExportFormat(str, Enum):
     html = "html"
     go = "go"
     bigquery = "bigquery"
+    dbml = "dbml"
 
 
 @app.command()
@@ -229,10 +228,17 @@ class ImportFormat(str, Enum):
 @app.command(name="import")
 def import_(
     format: Annotated[ImportFormat, typer.Option(help="The format of the source file.")],
-    source: Annotated[Optional[str], typer.Option(help="The path to the file or Glue Database that should be imported.")] = None,
+    source: Annotated[
+        Optional[str], typer.Option(help="The path to the file or Glue Database that should be imported.")
+    ] = None,
     bigquery_project: Annotated[Optional[str], typer.Option(help="The bigquery project id.")] = None,
     bigquery_dataset: Annotated[Optional[str], typer.Option(help="The bigquery dataset id.")] = None,
-    bigquery_table: Annotated[Optional[List[str]], typer.Option(help="List of table ids to import from the bigquery API (repeat for multiple table ids, leave empty for all tables in the dataset).")] = None,
+    bigquery_table: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            help="List of table ids to import from the bigquery API (repeat for multiple table ids, leave empty for all tables in the dataset)."
+        ),
+    ] = None,
 ):
     """
     Create a data contract from the given source location. Prints to stdout.

datacontract/data_contract.py

@@ -15,16 +15,15 @@ from datacontract.engines.soda.check_soda_execute import check_soda_execute
 from datacontract.export.avro_converter import to_avro_schema_json
 from datacontract.export.avro_idl_converter import to_avro_idl
 from datacontract.export.bigquery_converter import to_bigquery_json
-from datacontract.export.dbt_converter import to_dbt_models_yaml, \
-    to_dbt_sources_yaml, to_dbt_staging_sql
-from datacontract.export.great_expectations_converter import \
-    to_great_expectations
+from datacontract.export.dbml_converter import to_dbml_diagram
+from datacontract.export.dbt_converter import to_dbt_models_yaml, to_dbt_sources_yaml, to_dbt_staging_sql
+from datacontract.export.go_converter import to_go_types
+from datacontract.export.great_expectations_converter import to_great_expectations
 from datacontract.export.html_export import to_html
 from datacontract.export.jsonschema_converter import to_jsonschema_json
 from datacontract.export.odcs_converter import to_odcs_yaml
 from datacontract.export.protobuf_converter import to_protobuf
 from datacontract.export.pydantic_converter import to_pydantic_model_str
-from datacontract.export.go_converter import to_go_types
 from datacontract.export.rdf_converter import to_rdf_n3
 from datacontract.export.sodacl_converter import to_sodacl_yaml
 from datacontract.export.sql_converter import to_sql_ddl, to_sql_query
@@ -32,10 +31,9 @@ from datacontract.export.terraform_converter import to_terraform
 from datacontract.imports.avro_importer import import_avro
 from datacontract.imports.bigquery_importer import import_bigquery_from_api, import_bigquery_from_json
 from datacontract.imports.glue_importer import import_glue
-from datacontract.imports.sql_importer import import_sql
 from datacontract.imports.jsonschema_importer import import_jsonschema
-from datacontract.integration.publish_datamesh_manager import \
-    publish_datamesh_manager
+from datacontract.imports.sql_importer import import_sql
+from datacontract.integration.publish_datamesh_manager import publish_datamesh_manager
 from datacontract.integration.publish_opentelemetry import publish_opentelemetry
 from datacontract.lint import resolve
 from datacontract.lint.linters.description_linter import DescriptionLinter
@@ -330,10 +328,17 @@ class DataContract:
             model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
             found_server = data_contract.servers.get(self._server)
             if found_server is None:
-                raise RuntimeError(f"Export to {export_format} requires selecting a bigquery server from the data contract.")
-            if found_server.type != 'bigquery':
-                raise RuntimeError(f"Export to {export_format} requires selecting a bigquery server from the data contract.")
+                raise RuntimeError(
+                    f"Export to {export_format} requires selecting a bigquery server from the data contract."
+                )
+            if found_server.type != "bigquery":
+                raise RuntimeError(
+                    f"Export to {export_format} requires selecting a bigquery server from the data contract."
+                )
             return to_bigquery_json(model_name, model_value, found_server)
+        if export_format == "dbml":
+            found_server = data_contract.servers.get(self._server)
+            return to_dbml_diagram(data_contract, found_server)
         else:
             print(f"Export format {export_format} not supported.")
             return ""
@@ -387,8 +392,10 @@ class DataContract:
         )
         run.log_info(f"Using {server} for testing the examples")
         return server
-    
-    def _check_models_for_export(self, data_contract: DataContractSpecification, model: str, export_format: str) -> typing.Tuple[str, str]:
+
+    def _check_models_for_export(
+        self, data_contract: DataContractSpecification, model: str, export_format: str
+    ) -> typing.Tuple[str, str]:
         if data_contract.models is None:
             raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
 
@@ -408,10 +415,17 @@ class DataContract:
                 raise RuntimeError(
                     f"Model {model_name} not found in the data contract. Available models: {model_names}"
                 )
-            
+
         return model_name, model_value
 
-    def import_from_source(self, format: str, source: typing.Optional[str] = None, bigquery_tables: typing.Optional[typing.List[str]] = None, bigquery_project: typing.Optional[str] = None, bigquery_dataset: typing.Optional[str] = None) -> DataContractSpecification:
+    def import_from_source(
+        self,
+        format: str,
+        source: typing.Optional[str] = None,
+        bigquery_tables: typing.Optional[typing.List[str]] = None,
+        bigquery_project: typing.Optional[str] = None,
+        bigquery_dataset: typing.Optional[str] = None,
+    ) -> DataContractSpecification:
         data_contract_specification = DataContract.init()
 
         if format == "sql":
@@ -426,7 +440,9 @@ class DataContract:
             if source is not None:
                 data_contract_specification = import_bigquery_from_json(data_contract_specification, source)
             else:
-                data_contract_specification = import_bigquery_from_api(data_contract_specification, bigquery_tables, bigquery_project, bigquery_dataset)
+                data_contract_specification = import_bigquery_from_api(
+                    data_contract_specification, bigquery_tables, bigquery_project, bigquery_dataset
+                )
         else:
             print(f"Import format {format} not supported.")
 

datacontract/engines/soda/check_soda_execute.py

@@ -9,6 +9,7 @@ from datacontract.engines.soda.connections.duckdb import get_duckdb_connection
 from datacontract.engines.soda.connections.kafka import create_spark_session, read_kafka_topic
 from datacontract.engines.soda.connections.postgres import to_postgres_soda_configuration
 from datacontract.engines.soda.connections.snowflake import to_snowflake_soda_configuration
+from datacontract.engines.soda.connections.sqlserver import to_sqlserver_soda_configuration
 from datacontract.export.sodacl_converter import to_sodacl_yaml
 from datacontract.model.data_contract_specification import DataContractSpecification, Server
 from datacontract.model.run import Run, Check, Log
@@ -69,6 +70,10 @@ def check_soda_execute(
         read_kafka_topic(spark, data_contract, server, tmp_dir)
         scan.add_spark_session(spark, data_source_name=server.type)
         scan.set_data_source_name(server.type)
+    elif server.type == "sqlserver":
+        soda_configuration_str = to_sqlserver_soda_configuration(server)
+        scan.add_configuration_yaml_str(soda_configuration_str)
+        scan.set_data_source_name(server.type)
 
     else:
         run.checks.append(

datacontract/engines/soda/connections/duckdb.py

@@ -87,8 +87,7 @@ def setup_s3_connection(con, server):
         s3_endpoint = server.endpointUrl.removeprefix("http://").removeprefix("https://")
         if server.endpointUrl.startswith("http://"):
             use_ssl = "false"
-            url_style = 'path'
-
+            url_style = "path"
 
     if s3_access_key_id is not None:
         con.sql(f"""

datacontract/engines/soda/connections/sqlserver.py

@@ -0,0 +1,43 @@
+import os
+
+import yaml
+
+from datacontract.model.data_contract_specification import Server
+
+
+def to_sqlserver_soda_configuration(server: Server) -> str:
+    """Serialize server config to soda configuration.
+
+
+    ### Example:
+        type: sqlserver
+        host: host
+        port: '1433'
+        username: simple
+        password: simple_pass
+        database: database
+        schema: dbo
+        trusted_connection: false
+        encrypt: false
+        trust_server_certificate: false
+        driver: ODBC Driver 18 for SQL Server
+    """
+    # with service account key, using an external json file
+    soda_configuration = {
+        f"data_source {server.type}": {
+            "type": "sqlserver",
+            "host": server.host,
+            "port": str(server.port),
+            "username": os.getenv("DATACONTRACT_SQLSERVER_USERNAME", ""),
+            "password": os.getenv("DATACONTRACT_SQLSERVER_PASSWORD", ""),
+            "database": server.database,
+            "schema": server.schema_,
+            "trusted_connection": os.getenv("DATACONTRACT_SQLSERVER_TRUSTED_CONNECTION", False),
+            "trust_server_certificate": os.getenv("DATACONTRACT_SQLSERVER_TRUST_SERVER_CERTIFICATE", False),
+            "encrypt": os.getenv("DATACONTRACT_SQLSERVER_ENCRYPTED_CONNECTION", True),
+            "driver": server.driver,
+        }
+    }
+
+    soda_configuration_str = yaml.dump(soda_configuration)
+    return soda_configuration_str

datacontract/export/avro_converter.py

@@ -38,6 +38,12 @@ def to_avro_field(field, field_name):
     if field.type in ["timestamp", "timestamp_tz", "timestamp_ntz", "date"]:
         avro_field["logicalType"] = to_avro_logical_type(field.type)
 
+    if field.config:
+        if "avroLogicalType" in field.config:
+            avro_field["logicalType"] = field.config["avroLogicalType"]
+        if "avroDefault" in field.config:
+            avro_field["default"] = field.config["avroDefault"]
+
     return avro_field
 
 
@@ -77,6 +83,7 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
     else:
         return "bytes"
 
+
 def to_avro_logical_type(type: str) -> str:
     if type in ["timestamp", "timestamp_tz"]:
         return "timestamp-millis"
@@ -85,4 +92,4 @@ def to_avro_logical_type(type: str) -> str:
     elif type in ["date"]:
         return "date"
     else:
-        return ""
+        return ""

datacontract/export/bigquery_converter.py

@@ -5,24 +5,21 @@ from typing import Dict, List
 from datacontract.model.data_contract_specification import Model, Field, Server
 from datacontract.model.exceptions import DataContractException
 
+
 def to_bigquery_json(model_name: str, model_value: Model, server: Server) -> str:
     bigquery_table = to_bigquery_schema(model_name, model_value, server)
     return json.dumps(bigquery_table, indent=2)
 
+
 def to_bigquery_schema(model_name: str, model_value: Model, server: Server) -> dict:
     return {
         "kind": "bigquery#table",
-        "tableReference": {
-            "datasetId": server.dataset,
-            "projectId": server.project,
-            "tableId": model_name
-        },
+        "tableReference": {"datasetId": server.dataset, "projectId": server.project, "tableId": model_name},
         "description": model_value.description,
-        "schema": {
-            "fields": to_fields_array(model_value.fields)
-            }
+        "schema": {"fields": to_fields_array(model_value.fields)},
     }
 
+
 def to_fields_array(fields: Dict[str, Field]) -> List[Dict[str, Field]]:
     bq_fields = []
     for field_name, field in fields.items():
@@ -32,24 +29,25 @@ def to_fields_array(fields: Dict[str, Field]) -> List[Dict[str, Field]]:
 
 
 def to_field(field_name: str, field: Field) -> dict:
-
     bq_type = map_type_to_bigquery(field.type, field_name)
     bq_field = {
         "name": field_name,
         "type": bq_type,
         "mode": "REQUIRED" if field.required else "NULLABLE",
-        "description": field.description
+        "description": field.description,
     }
 
     # handle arrays
-    if field.type == 'array':
-        bq_field["mode"] = 'REPEATED'
-        if field.items.type == 'object':
+    if field.type == "array":
+        bq_field["mode"] = "REPEATED"
+        if field.items.type == "object":
             # in case the array type is a complex object, we want to copy all its fields
             bq_field["fields"] = to_fields_array(field.items.fields)
         else:
             # otherwise we make up a structure that gets us a single field of the specified type
-            bq_field["fields"] = to_fields_array({ f"{field_name}_1": Field(type=field.items.type, required=False, description="")})
+            bq_field["fields"] = to_fields_array(
+                {f"{field_name}_1": Field(type=field.items.type, required=False, description="")}
+            )
     # all of these can carry other fields
     elif bq_type.lower() in ["record", "struct"]:
         bq_field["fields"] = to_fields_array(field.fields)
@@ -65,6 +63,7 @@ def to_field(field_name: str, field: Field) -> dict:
 
     return bq_field
 
+
 def map_type_to_bigquery(type_str: str, field_name: str) -> str:
     logger = logging.getLogger(__name__)
     if type_str.lower() in ["string", "varchar", "text"]:
@@ -78,7 +77,7 @@ def map_type_to_bigquery(type_str: str, field_name: str) -> str:
     elif type_str == "float":
         return "FLOAT"
     elif type_str == "boolean":
-        return "BOOLEAN"
+        return "BOOL"
     elif type_str.lower() in ["timestamp", "timestamp_tz"]:
         return "TIMESTAMP"
     elif type_str == "date":
@@ -94,7 +93,9 @@ def map_type_to_bigquery(type_str: str, field_name: str) -> str:
     elif type_str == "struct":
         return "STRUCT"
     elif type_str == "null":
-        logger.info(f"Can't properly map {field_name} to bigquery Schema, as 'null' is not supported as a type. Mapping it to STRING.")
+        logger.info(
+            f"Can't properly map {field_name} to bigquery Schema, as 'null' is not supported as a type. Mapping it to STRING."
+        )
         return "STRING"
     else:
         raise DataContractException(

datacontract/export/dbml_converter.py

@@ -0,0 +1,118 @@
+from datetime import datetime
+from importlib.metadata import version
+from typing import Tuple
+
+import pytz
+
+import datacontract.model.data_contract_specification as spec
+from datacontract.export.sql_type_converter import convert_to_sql_type
+
+
+def to_dbml_diagram(contract: spec.DataContractSpecification, server: spec.Server) -> str:
+    result = ""
+    result += add_generated_info(contract, server) + "\n"
+    result += generate_project_info(contract) + "\n"
+
+    for model_name, model in contract.models.items():
+        table_description = generate_table(model_name, model, server)
+        result += f"\n{table_description}\n"
+
+    return result
+
+
+def add_generated_info(contract: spec.DataContractSpecification, server: spec.Server) -> str:
+    tz = pytz.timezone("UTC")
+    now = datetime.now(tz)
+    formatted_date = now.strftime("%b %d %Y")
+    datacontract_cli_version = get_version()
+    dialect = "Logical Datacontract" if server is None else server.type
+
+    generated_info = """
+Generated at {0} by datacontract-cli version {1}
+for datacontract {2} ({3}) version {4} 
+Using {5} Types for the field types
+    """.format(
+        formatted_date, datacontract_cli_version, contract.info.title, contract.id, contract.info.version, dialect
+    )
+
+    comment = """/*
+{0}
+*/
+    """.format(generated_info)
+
+    note = """Note project_info {{
+'''
+{0}
+'''
+}}
+    """.format(generated_info)
+
+    return """{0}
+{1}
+    """.format(comment, note)
+
+
+def get_version() -> str:
+    try:
+        return version("datacontract_cli")
+    except Exception:
+        return ""
+
+
+def generate_project_info(contract: spec.DataContractSpecification) -> str:
+    return """Project "{0}" {{
+    Note: "{1}"
+}}\n
+    """.format(contract.info.title, " ".join(contract.info.description.splitlines()))
+
+
+def generate_table(model_name: str, model: spec.Model, server: spec.Server) -> str:
+    result = """Table "{0}" {{ 
+Note: "{1}"
+    """.format(model_name, " ".join(model.description.splitlines()))
+
+    references = []
+
+    # Add all the fields
+    for field_name, field in model.fields.items():
+        ref, field_string = generate_field(field_name, field, model_name, server)
+        if ref is not None:
+            references.append(ref)
+        result += "{0}\n".format(field_string)
+
+    result += "}\n"
+
+    # and if any: add the references
+    if len(references) > 0:
+        for ref in references:
+            result += "Ref: {0}\n".format(ref)
+
+        result += "\n"
+
+    return result
+
+
+def generate_field(field_name: str, field: spec.Field, model_name: str, server: spec.Server) -> Tuple[str, str]:
+    field_attrs = []
+    if field.primary:
+        field_attrs.append("pk")
+
+    if field.unique:
+        field_attrs.append("unique")
+
+    if field.required:
+        field_attrs.append("not null")
+    else:
+        field_attrs.append("null")
+
+    if field.description:
+        field_attrs.append('Note: "{0}"'.format(" ".join(field.description.splitlines())))
+
+    field_type = field.type if server is None else convert_to_sql_type(field, server.type)
+
+    field_str = '"{0}" "{1}" [{2}]'.format(field_name, field_type, ",".join(field_attrs))
+    ref_str = None
+    if (field.references) is not None:
+        # we always assume many to one, as datacontract doesn't really give us more info
+        ref_str = "{0}.{1} > {2}".format(model_name, field_name, field.references)
+    return (ref_str, field_str)

datacontract/export/html_export.py

@@ -2,13 +2,12 @@ import datetime
 import logging
 from importlib.metadata import version
 
+import jinja_partials
 import pytz
 import yaml
-import jinja_partials
 from jinja2 import Environment, PackageLoader, select_autoescape
 
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 def to_html(data_contract_spec: DataContractSpecification) -> str:

datacontract/export/jsonschema_converter.py

@@ -18,12 +18,11 @@ def to_jsonschema_json(model_key, model_value: Model) -> str:
 
 
 def to_jsonschema(model_key, model_value: Model) -> dict:
-
-    model =  {
+    model = {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "type": "object",
         "properties": to_properties(model_value.fields),
-        "required": to_required(model_value.fields)
+        "required": to_required(model_value.fields),
     }
     if model_value.title:
         model["title"] = model_value.title
@@ -54,7 +53,7 @@ def to_property(field: Field) -> dict:
         property["unique"] = True
     if json_type == "object":
         # TODO: any better idea to distinguish between properties and patternProperties?
-        if next(iter(field.fields.keys())).startswith("^"):
+        if field.fields.keys() and next(iter(field.fields.keys())).startswith("^"):
             property["patternProperties"] = to_properties(field.fields)
         else:
             property["properties"] = to_properties(field.fields)
@@ -89,10 +88,10 @@ def to_property(field: Field) -> dict:
     if field.classification:
         property["classification"] = field.classification
 
-    
     # TODO: all constraints
     return property
 
+
 def to_required(fields: Dict[str, Field]):
     required = []
     for field_name, field in fields.items():

datacontract/export/rdf_converter.py

@@ -1,8 +1,7 @@
 from pydantic import BaseModel
 from rdflib import Graph, Literal, BNode, RDF, URIRef, Namespace
 
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
+from datacontract.model.data_contract_specification import DataContractSpecification
 
 
 def is_literal(property_name):