datacontract-cli 0.10.7__py3-none-any.whl → 0.10.8__py3-none-any.whl

datacontract/cli.py

@@ -5,6 +5,7 @@ from typing import Iterable, Optional
 from typing import List
 
 import typer
+import uvicorn
 from click import Context
 from rich import box
 from rich.console import Console
@@ -12,6 +13,7 @@ from rich.table import Table
 from typer.core import TyperGroup
 from typing_extensions import Annotated
 
+from datacontract import web
 from datacontract.catalog.catalog import create_index_html, create_data_contract_html
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
@@ -183,6 +185,7 @@ def export(
     result = DataContract(data_contract_file=location, server=server).export(
         export_format=format,
         model=model,
+        server=server,
         rdf_base=rdf_base,
         sql_server_type=sql_server_type,
     )
@@ -201,6 +204,8 @@ class ImportFormat(str, Enum):
     glue = "glue"
     bigquery = "bigquery"
     jsonschema = "jsonschema"
+    odcs="odcs"
+    unity = "unity"
 
 
 @app.command(name="import")
@@ -223,11 +228,12 @@ def import_(
             help="List of table ids to import from the bigquery API (repeat for multiple table ids, leave empty for all tables in the dataset)."
         ),
     ] = None,
+    unity_table_full_name: Annotated[Optional[str], typer.Option(help="Full name of a table in the unity catalog")] = None,
 ):
     """
     Create a data contract from the given source location. Prints to stdout.
     """
-    result = DataContract().import_from_source(format, source, glue_table, bigquery_table, bigquery_project, bigquery_dataset)
+    result = DataContract().import_from_source(format, source, glue_table, bigquery_table, bigquery_project, bigquery_dataset, unity_table_full_name)
     console.print(result.to_yaml())
 
 
@@ -323,6 +329,18 @@ def diff(
     console.print(result.changelog_str())
 
 
+@app.command()
+def serve(
+    port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4242,
+    host: Annotated[str, typer.Option(help="Bind socket to this host.")] = "127.0.0.1",
+):
+    """
+    Start the datacontract web server.
+    """
+
+    uvicorn.run(web.app, port=port, host=host)
+
+
 def _handle_result(run):
     _print_table(run)
     if run.result == "passed":

datacontract/data_contract.py

@@ -2,7 +2,6 @@ import json
 import logging
 import tempfile
 import typing
-from enum import Enum
 
 import yaml
 from pyspark.sql import SparkSession
@@ -13,27 +12,15 @@ from datacontract.engines.datacontract.check_that_datacontract_contains_valid_se
 )
 from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
 from datacontract.engines.soda.check_soda_execute import check_soda_execute
-from datacontract.export.avro_converter import to_avro_schema_json
-from datacontract.export.avro_idl_converter import to_avro_idl
-from datacontract.export.bigquery_converter import to_bigquery_json
-from datacontract.export.dbml_converter import to_dbml_diagram
-from datacontract.export.dbt_converter import to_dbt_models_yaml, to_dbt_sources_yaml, to_dbt_staging_sql
-from datacontract.export.go_converter import to_go_types
-from datacontract.export.great_expectations_converter import to_great_expectations
-from datacontract.export.html_export import to_html
-from datacontract.export.jsonschema_converter import to_jsonschema_json
-from datacontract.export.odcs_converter import to_odcs_yaml
-from datacontract.export.protobuf_converter import to_protobuf
-from datacontract.export.pydantic_converter import to_pydantic_model_str
-from datacontract.export.rdf_converter import to_rdf_n3
-from datacontract.export.sodacl_converter import to_sodacl_yaml
-from datacontract.export.sql_converter import to_sql_ddl, to_sql_query
-from datacontract.export.terraform_converter import to_terraform
+from datacontract.export.exporter import ExportFormat
+from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.avro_importer import import_avro
 from datacontract.imports.bigquery_importer import import_bigquery_from_api, import_bigquery_from_json
 from datacontract.imports.glue_importer import import_glue
 from datacontract.imports.jsonschema_importer import import_jsonschema
+from datacontract.imports.odcs_importer import import_odcs
 from datacontract.imports.sql_importer import import_sql
+from datacontract.imports.unity_importer import import_unity_from_json, import_unity_from_api
 from datacontract.integration.publish_datamesh_manager import publish_datamesh_manager
 from datacontract.integration.publish_opentelemetry import publish_opentelemetry
 from datacontract.lint import resolve
@@ -50,28 +37,6 @@ from datacontract.model.exceptions import DataContractException
 from datacontract.model.run import Run, Check
 
 
-class ExportFormat(str, Enum):
-    jsonschema = "jsonschema"
-    pydantic_model = "pydantic-model"
-    sodacl = "sodacl"
-    dbt = "dbt"
-    dbt_sources = "dbt-sources"
-    dbt_staging_sql = "dbt-staging-sql"
-    odcs = "odcs"
-    rdf = "rdf"
-    avro = "avro"
-    protobuf = "protobuf"
-    great_expectations = "great-expectations"
-    terraform = "terraform"
-    avro_idl = "avro-idl"
-    sql = "sql"
-    sql_query = "sql-query"
-    html = "html"
-    go = "go"
-    bigquery = "bigquery"
-    dbml = "dbml"
-
-
 class DataContract:
     def __init__(
         self,
@@ -207,6 +172,9 @@ class DataContract:
                 if self._examples:
                     server_name = "examples"
                     server = self._get_examples_server(data_contract, run, tmp_dir)
+                elif self._server:
+                    server_name = self._server
+                    server = data_contract.servers.get(server_name)
                 else:
                     server_name = list(data_contract.servers.keys())[0]
                     server = data_contract.servers.get(server_name)
@@ -260,6 +228,38 @@ class DataContract:
 
         return run
 
+    def _get_examples_server(self, data_contract, run, tmp_dir):
+        run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
+        format = "json"
+        for example in data_contract.examples:
+            format = example.type
+            p = f"{tmp_dir}/{example.model}.{format}"
+            run.log_info(f"Creating example file {p}")
+            with open(p, "w") as f:
+                content = ""
+                if format == "json" and isinstance(example.data, list):
+                    content = json.dumps(example.data)
+                elif format == "json" and isinstance(example.data, str):
+                    content = example.data
+                elif format == "yaml" and isinstance(example.data, list):
+                    content = yaml.dump(example.data, allow_unicode=True)
+                elif format == "yaml" and isinstance(example.data, str):
+                    content = example.data
+                elif format == "csv":
+                    content = example.data
+                logging.debug(f"Content of example file {p}: {content}")
+                f.write(content)
+        path = f"{tmp_dir}" + "/{model}." + format
+        delimiter = "array"
+        server = Server(
+            type="local",
+            path=path,
+            format=format,
+            delimiter=delimiter,
+        )
+        run.log_info(f"Using {server} for testing the examples")
+        return server
+
     def breaking(self, other: "DataContract") -> BreakingChanges:
         return self.changelog(other, include_severities=[Severity.ERROR, Severity.WARNING])
 
@@ -302,7 +302,11 @@ class DataContract:
         )
 
     def export(
-        self, export_format: ExportFormat, model: str = "all", rdf_base: str = None, sql_server_type: str = "auto"
+        self,
+        export_format: ExportFormat,
+        model: str = "all",
+        sql_server_type: str = "auto",
+        **kwargs,
     ) -> str:
         data_contract = resolve.resolve_data_contract(
             self._data_contract_file,
@@ -311,140 +315,14 @@ class DataContract:
             inline_definitions=True,
             inline_quality=True,
         )
-        if export_format == "jsonschema":
-            model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
-            return to_jsonschema_json(model_name, model_value)
-        if export_format == "sodacl":
-            return to_sodacl_yaml(data_contract)
-        if export_format == "dbt":
-            return to_dbt_models_yaml(data_contract)
-        if export_format == "dbt-sources":
-            return to_dbt_sources_yaml(data_contract, self._server)
-        if export_format == "dbt-staging-sql":
-            model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
-            return to_dbt_staging_sql(data_contract, model_name, model_value)
-        if export_format == "odcs":
-            return to_odcs_yaml(data_contract)
-        if export_format == "rdf":
-            return to_rdf_n3(data_contract, rdf_base)
-        if export_format == "protobuf":
-            return to_protobuf(data_contract)
-        if export_format == "avro":
-            model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
-            return to_avro_schema_json(model_name, model_value)
-        if export_format == "avro-idl":
-            return to_avro_idl(data_contract)
-        if export_format == "terraform":
-            return to_terraform(data_contract)
-        if export_format == "sql":
-            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
-            return to_sql_ddl(data_contract, server_type=server_type)
-        if export_format == "sql-query":
-            model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
-            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
-            return to_sql_query(data_contract, model_name, model_value, server_type)
-        if export_format == "great-expectations":
-            model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
-            return to_great_expectations(data_contract, model_name)
-        if export_format == "pydantic-model":
-            return to_pydantic_model_str(data_contract)
-        if export_format == "html":
-            return to_html(data_contract)
-        if export_format == "go":
-            return to_go_types(data_contract)
-        if export_format == "bigquery":
-            model_name, model_value = self._check_models_for_export(data_contract, model, export_format)
-            found_server = data_contract.servers.get(self._server)
-            if found_server is None:
-                raise RuntimeError(
-                    f"Export to {export_format} requires selecting a bigquery server from the data contract."
-                )
-            if found_server.type != "bigquery":
-                raise RuntimeError(
-                    f"Export to {export_format} requires selecting a bigquery server from the data contract."
-                )
-            return to_bigquery_json(model_name, model_value, found_server)
-        if export_format == "dbml":
-            found_server = data_contract.servers.get(self._server)
-            return to_dbml_diagram(data_contract, found_server)
-        else:
-            print(f"Export format {export_format} not supported.")
-            return ""
-
-    def _determine_sql_server_type(self, data_contract: DataContractSpecification, sql_server_type: str):
-        if sql_server_type == "auto":
-            if data_contract.servers is None or len(data_contract.servers) == 0:
-                raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
-
-            server_types = set([server.type for server in data_contract.servers.values()])
-            if "snowflake" in server_types:
-                return "snowflake"
-            elif "postgres" in server_types:
-                return "postgres"
-            elif "databricks" in server_types:
-                return "databricks"
-            else:
-                # default to snowflake dialect
-                return "snowflake"
-        else:
-            return sql_server_type
 
-    def _get_examples_server(self, data_contract, run, tmp_dir):
-        run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
-        format = "json"
-        for example in data_contract.examples:
-            format = example.type
-            p = f"{tmp_dir}/{example.model}.{format}"
-            run.log_info(f"Creating example file {p}")
-            with open(p, "w") as f:
-                content = ""
-                if format == "json" and isinstance(example.data, list):
-                    content = json.dumps(example.data)
-                elif format == "json" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "yaml" and isinstance(example.data, list):
-                    content = yaml.dump(example.data, allow_unicode=True)
-                elif format == "yaml" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "csv":
-                    content = example.data
-                logging.debug(f"Content of example file {p}: {content}")
-                f.write(content)
-        path = f"{tmp_dir}" + "/{model}." + format
-        delimiter = "array"
-        server = Server(
-            type="local",
-            path=path,
-            format=format,
-            delimiter=delimiter,
+        return exporter_factory.create(export_format).export(
+            data_contract=data_contract,
+            model=model,
+            server=self._server,
+            sql_server_type=sql_server_type,
+            export_args=kwargs,
         )
-        run.log_info(f"Using {server} for testing the examples")
-        return server
-
-    def _check_models_for_export(
-        self, data_contract: DataContractSpecification, model: str, export_format: str
-    ) -> typing.Tuple[str, str]:
-        if data_contract.models is None:
-            raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
-
-        model_names = list(data_contract.models.keys())
-
-        if model == "all":
-            if len(data_contract.models.items()) != 1:
-                raise RuntimeError(
-                    f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
-                )
-
-            model_name, model_value = next(iter(data_contract.models.items()))
-        else:
-            model_name = model
-            model_value = data_contract.models.get(model_name)
-            if model_value is None:
-                raise RuntimeError(
-                    f"Model {model_name} not found in the data contract. Available models: {model_names}"
-                )
-
-        return model_name, model_value
 
     def import_from_source(
         self,
@@ -454,6 +332,7 @@ class DataContract:
         bigquery_tables: typing.Optional[typing.List[str]] = None,
         bigquery_project: typing.Optional[str] = None,
         bigquery_dataset: typing.Optional[str] = None,
+        unity_table_full_name: typing.Optional[str] = None
     ) -> DataContractSpecification:
         data_contract_specification = DataContract.init()
 
@@ -472,6 +351,15 @@ class DataContract:
                 data_contract_specification = import_bigquery_from_api(
                     data_contract_specification, bigquery_tables, bigquery_project, bigquery_dataset
                 )
+        elif format == "odcs":
+            data_contract_specification = import_odcs(data_contract_specification, source)
+        elif format == "unity":
+            if source is not None:
+                data_contract_specification = import_unity_from_json(data_contract_specification, source)
+            else:
+                data_contract_specification = import_unity_from_api(
+                    data_contract_specification, unity_table_full_name
+                )
         else:
             print(f"Import format {format} not supported.")
 

datacontract/engines/fastjsonschema/s3/s3_read_files.py

@@ -1,7 +1,7 @@
 import logging
 import os
 
-import s3fs
+from datacontract.model.exceptions import DataContractException
 
 
 def yield_s3_files(s3_endpoint_url, s3_location):
@@ -14,6 +14,18 @@ def yield_s3_files(s3_endpoint_url, s3_location):
 
 
 def s3_fs(s3_endpoint_url):
+    try:
+        import s3fs
+    except ImportError as e:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="s3 extra missing",
+            reason="Install the extra datacontract-cli\[s3] to use s3",
+            engine="datacontract",
+            original_exception=e,
+        )
+
     aws_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")
     aws_secret_access_key = os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY")
     return s3fs.S3FileSystem(

datacontract/engines/soda/check_soda_execute.py

@@ -66,8 +66,10 @@ def check_soda_execute(
             scan.set_data_source_name(server.type)
     elif server.type == "dataframe":
         if spark is None:
-            run.log_warn("Server type dataframe only works with the Python library and requires a Spark session, "
-                         "please provide one with the DataContract class")
+            run.log_warn(
+                "Server type dataframe only works with the Python library and requires a Spark session, "
+                "please provide one with the DataContract class"
+            )
             return
         else:
             logging.info("Use Spark to connect to data source")

datacontract/engines/soda/connections/bigquery.py

@@ -6,10 +6,17 @@ import yaml
 # https://docs.soda.io/soda/connect-bigquery.html#authentication-methods
 def to_bigquery_soda_configuration(server):
     # with service account key, using an external json file
+
+    # check for our own environment variable first
+    account_info = os.getenv("DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH")
+    if account_info is None:
+        # but as a fallback look for the default google one
+        account_info = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
+
     soda_configuration = {
         f"data_source {server.type}": {
             "type": "bigquery",
-            "account_info_json_path": os.getenv("DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH"),
+            "account_info_json_path": account_info,
             "auth_scopes": ["https://www.googleapis.com/auth/bigquery"],
             "project_id": server.project,
             "dataset": server.dataset,

datacontract/export/avro_converter.py

@@ -1,8 +1,15 @@
 import json
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
 from datacontract.model.data_contract_specification import Field
 
 
+class AvroExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_avro_schema_json(model_name, model_value)
+
+
 def to_avro_schema(model_name, model) -> dict:
     return to_avro_record(model_name, model.fields, model.description, model.namespace)
 
@@ -47,7 +54,13 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
         if "avroLogicalType" in field.config and "avroType" in field.config:
             return {"type": field.config["avroType"], "logicalType": field.config["avroLogicalType"]}
         if "avroLogicalType" in field.config:
-            if field.config["avroLogicalType"] in ["timestamp-millis", "timestamp-micros", "local-timestamp-millis", "local-timestamp-micros", "time-micros"]:
+            if field.config["avroLogicalType"] in [
+                "timestamp-millis",
+                "timestamp-micros",
+                "local-timestamp-millis",
+                "local-timestamp-micros",
+                "time-micros",
+            ]:
                 return {"type": "long", "logicalType": field.config["avroLogicalType"]}
             if field.config["avroLogicalType"] in ["time-millis", "date"]:
                 return {"type": "int", "logicalType": field.config["avroLogicalType"]}
@@ -82,8 +95,7 @@ def to_avro_type(field: Field, field_name: str) -> str | dict:
     elif field.type in ["binary"]:
         return "bytes"
     elif field.type in ["array"]:
-        # TODO support array structs
-        return "array"
+        return {"type": "array", "items": to_avro_type(field.items, field_name)}
     elif field.type in ["null"]:
         return "null"
     else:

datacontract/export/avro_idl_converter.py

@@ -7,28 +7,7 @@ from datacontract.lint.resolve import inline_definitions_into_data_contract
 from datacontract.model.data_contract_specification import DataContractSpecification, Field
 from datacontract.model.exceptions import DataContractException
 
-
-def to_avro_idl(contract: DataContractSpecification) -> str:
-    """Serialize the provided data contract specification into an Avro IDL string.
-
-    The data contract will be serialized as a protocol, with one record type
-    for each contained model. Model fields are mapped one-to-one to Avro IDL
-    record fields.
-    """
-    stream = StringIO()
-    to_avro_idl_stream(contract, stream)
-    return stream.getvalue()
-
-
-def to_avro_idl_stream(contract: DataContractSpecification, stream: typing.TextIO):
-    """Serialize the provided data contract specification into Avro IDL."""
-    ir = _contract_to_avro_idl_ir(contract)
-    if ir.description:
-        stream.write(f"/** {contract.info.description} */\n")
-    stream.write(f"protocol {ir.name or 'Unnamed'} {{\n")
-    for model_type in ir.model_types:
-        _write_model_type(model_type, stream)
-    stream.write("}\n")
+from datacontract.export.exporter import Exporter
 
 
 class AvroPrimitiveType(Enum):
@@ -107,6 +86,34 @@ avro_primitive_types = set(
 )
 
 
+class AvroIdlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_avro_idl(data_contract)
+
+
+def to_avro_idl(contract: DataContractSpecification) -> str:
+    """Serialize the provided data contract specification into an Avro IDL string.
+
+    The data contract will be serialized as a protocol, with one record type
+    for each contained model. Model fields are mapped one-to-one to Avro IDL
+    record fields.
+    """
+    stream = StringIO()
+    to_avro_idl_stream(contract, stream)
+    return stream.getvalue()
+
+
+def to_avro_idl_stream(contract: DataContractSpecification, stream: typing.TextIO):
+    """Serialize the provided data contract specification into Avro IDL."""
+    ir = _contract_to_avro_idl_ir(contract)
+    if ir.description:
+        stream.write(f"/** {contract.info.description} */\n")
+    stream.write(f"protocol {ir.name or 'Unnamed'} {{\n")
+    for model_type in ir.model_types:
+        _write_model_type(model_type, stream)
+    stream.write("}\n")
+
+
 def _to_avro_primitive_logical_type(field_name: str, field: Field) -> AvroPrimitiveField:
     result = AvroPrimitiveField(field_name, field.required, field.description, AvroPrimitiveType.string)
     match field.type:

datacontract/export/bigquery_converter.py

@@ -5,6 +5,21 @@ from typing import Dict, List
 from datacontract.model.data_contract_specification import Model, Field, Server
 from datacontract.model.exceptions import DataContractException
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class BigQueryExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        self.dict_args = export_args
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        found_server = data_contract.servers.get(server)
+        if found_server is None:
+            raise RuntimeError("Export to bigquery requires selecting a bigquery server from the data contract.")
+        if found_server.type != "bigquery":
+            raise RuntimeError("Export to bigquery requires selecting a bigquery server from the data contract.")
+
+        return to_bigquery_json(model_name, model_value, found_server)
+
 
 def to_bigquery_json(model_name: str, model_value: Model, server: Server) -> str:
     bigquery_table = to_bigquery_schema(model_name, model_value, server)

datacontract/export/dbml_converter.py

@@ -8,6 +8,15 @@ import datacontract.model.data_contract_specification as spec
 from datacontract.export.sql_type_converter import convert_to_sql_type
 
 
+from datacontract.export.exporter import Exporter
+
+
+class DbmlExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        found_server = data_contract.servers.get(server)
+        return to_dbml_diagram(data_contract, found_server)
+
+
 def to_dbml_diagram(contract: spec.DataContractSpecification, server: spec.Server) -> str:
     result = ""
     result += add_generated_info(contract, server) + "\n"

datacontract/export/dbt_converter.py

@@ -5,6 +5,28 @@ import yaml
 from datacontract.export.sql_type_converter import convert_to_sql_type
 from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
 
+from datacontract.export.exporter import Exporter, _check_models_for_export
+
+
+class DbtExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_dbt_models_yaml(data_contract)
+
+
+class DbtSourceExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        return to_dbt_sources_yaml(data_contract, server)
+
+
+class DbtStageExporter(Exporter):
+    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
+        model_name, model_value = _check_models_for_export(data_contract, model, self.export_format)
+        return to_dbt_staging_sql(
+            data_contract,
+            model_name,
+            model_value,
+        )
+
 
 def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
     dbt = {
@@ -19,7 +41,10 @@ def to_dbt_models_yaml(data_contract_spec: DataContractSpecification):
 
 def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name: str, model_value: Model) -> str:
     if data_contract_spec.models is None or len(data_contract_spec.models.items()) != 1:
-        print("Export to dbt-staging-sql currently only works with exactly one model in the data contract.")
+        print(
+            "Export to dbt-staging-sql currently only works with exactly one model in the data contract."
+            "Please specify the model name."
+        )
         return ""
 
     id = data_contract_spec.id