datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/data_contract.py

@@ -1,56 +1,34 @@
-import json
 import logging
-import tempfile
 import typing
 
-import yaml
-from pyspark.sql import SparkSession
+from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard
 
-from datacontract.breaking.breaking import models_breaking_changes, \
-    quality_breaking_changes
-from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
-    check_that_datacontract_contains_valid_server_configuration,
+from datacontract.export.odcs_v3_exporter import to_odcs_v3
+from datacontract.imports.importer import ImportFormat, Spec
+from datacontract.imports.odcs_v3_importer import import_from_odcs
+
+if typing.TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
+from duckdb.duckdb import DuckDBPyConnection
+
+from datacontract.breaking.breaking import (
+    info_breaking_changes,
+    models_breaking_changes,
+    quality_breaking_changes,
+    terms_breaking_changes,
 )
-from datacontract.engines.fastjsonschema.check_jsonschema import \
-    check_jsonschema
-from datacontract.engines.soda.check_soda_execute import check_soda_execute
-from datacontract.export.avro_converter import to_avro_schema_json
-from datacontract.export.avro_idl_converter import to_avro_idl
-from datacontract.export.dbt_converter import to_dbt_models_yaml, \
-    to_dbt_sources_yaml, to_dbt_staging_sql
-from datacontract.export.great_expectations_converter import \
-    to_great_expectations
-from datacontract.export.html_export import to_html
-from datacontract.export.jsonschema_converter import to_jsonschema_json
-from datacontract.export.odcs_converter import to_odcs_yaml
-from datacontract.export.protobuf_converter import to_protobuf
-from datacontract.export.pydantic_converter import to_pydantic_model_str
-from datacontract.export.rdf_converter import to_rdf_n3
-from datacontract.export.sodacl_converter import to_sodacl_yaml
-from datacontract.export.sql_converter import to_sql_ddl, to_sql_query
-from datacontract.export.terraform_converter import to_terraform
-from datacontract.imports.avro_importer import import_avro
-from datacontract.imports.sql_importer import import_sql
-from datacontract.integration.publish_datamesh_manager import \
-    publish_datamesh_manager
-from datacontract.integration.publish_opentelemetry import publish_opentelemetry
+from datacontract.breaking.breaking_change import BreakingChange, BreakingChanges, Severity
+from datacontract.engines.data_contract_test import execute_data_contract_test
+from datacontract.export.exporter import ExportFormat
+from datacontract.export.exporter_factory import exporter_factory
+from datacontract.imports.importer_factory import importer_factory
+from datacontract.init.init_template import get_init_template
+from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
 from datacontract.lint import resolve
-from datacontract.lint.linters.description_linter import DescriptionLinter
-from datacontract.lint.linters.example_model_linter import ExampleModelLinter
-from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
-from datacontract.lint.linters.field_reference_linter import \
-    FieldReferenceLinter
-from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
-from datacontract.lint.linters.quality_schema_linter import \
-    QualityUsesSchemaLinter
-from datacontract.lint.linters.valid_constraints_linter import \
-    ValidFieldConstraintsLinter
-from datacontract.model.breaking_change import BreakingChanges, BreakingChange, \
-    Severity
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Server
+from datacontract.model.data_contract_specification import DataContractSpecification, Info
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Run, Check
+from datacontract.model.run import Check, ResultEnum, Run
 
 
 class DataContract:
@@ -61,41 +39,34 @@ class DataContract:
         data_contract: DataContractSpecification = None,
         schema_location: str = None,
         server: str = None,
-        examples: bool = False,
         publish_url: str = None,
-        publish_to_opentelemetry: bool = False,
-        spark: SparkSession = None,
-        inline_definitions: bool = False,
+        spark: "SparkSession" = None,
+        duckdb_connection: DuckDBPyConnection = None,
+        inline_definitions: bool = True,
+        inline_quality: bool = True,
+        ssl_verification: bool = True,
+        publish_test_results: bool = False,
     ):
         self._data_contract_file = data_contract_file
         self._data_contract_str = data_contract_str
         self._data_contract = data_contract
         self._schema_location = schema_location
         self._server = server
-        self._examples = examples
         self._publish_url = publish_url
-        self._publish_to_opentelemetry = publish_to_opentelemetry
+        self._publish_test_results = publish_test_results
         self._spark = spark
+        self._duckdb_connection = duckdb_connection
         self._inline_definitions = inline_definitions
-        self.all_linters = {
-            ExampleModelLinter(),
-            QualityUsesSchemaLinter(),
-            FieldPatternLinter(),
-            FieldReferenceLinter(),
-            NoticePeriodLinter(),
-            ValidFieldConstraintsLinter(),
-            DescriptionLinter(),
-        }
+        self._inline_quality = inline_quality
+        self._ssl_verification = ssl_verification
 
     @classmethod
-    def init(cls, template: str = "https://datacontract.com/datacontract.init.yaml") -> DataContractSpecification:
-        return resolve.resolve_data_contract(data_contract_location=template)
-
-    def lint(self, enabled_linters: typing.Union[str, set[str]] = "all") -> Run:
-        """Lint the data contract by deserializing the contract and checking the schema, as well as calling the configured linters.
+    def init(cls, template: typing.Optional[str], schema: typing.Optional[str] = None) -> DataContractSpecification:
+        template_str = get_init_template(template)
+        return resolve.resolve_data_contract(data_contract_str=template_str, schema_location=schema)
 
-        enabled_linters can be either "all" or "none", or a set of linter IDs. The "schema" linter is always enabled, even with enabled_linters="none".
-        """
+    def lint(self) -> Run:
+        """Lint the data contract by validating it against the JSON schema."""
         run = Run.create_run()
         try:
             run.log_info("Linting data contract")
@@ -104,32 +75,17 @@ class DataContract:
                 self._data_contract_str,
                 self._data_contract,
                 self._schema_location,
-                inline_definitions=True,
+                inline_definitions=self._inline_definitions,
+                inline_quality=self._inline_quality,
             )
             run.checks.append(
-                Check(type="lint", result="passed", name="Data contract is syntactically valid", engine="datacontract")
+                Check(
+                    type="lint",
+                    result=ResultEnum.passed,
+                    name="Data contract is syntactically valid",
+                    engine="datacontract",
+                )
             )
-            if enabled_linters == "none":
-                linters_to_check = set()
-            elif enabled_linters == "all":
-                linters_to_check = self.all_linters
-            elif isinstance(enabled_linters, set):
-                linters_to_check = {linter for linter in self.all_linters if linter.id in enabled_linters}
-            else:
-                raise RuntimeError(f"Unknown argument enabled_linters={enabled_linters} for lint()")
-            for linter in linters_to_check:
-                try:
-                    run.checks.extend(linter.lint(data_contract))
-                except Exception as e:
-                    run.checks.append(
-                        Check(
-                            type="general",
-                            result="error",
-                            name=f"Linter '{linter.name}'",
-                            reason=str(e),
-                            engine="datacontract",
-                        )
-                    )
             run.dataContractId = data_contract.id
             run.dataContractVersion = data_contract.info.version
         except DataContractException as e:
@@ -141,7 +97,7 @@ class DataContract:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Check Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -156,62 +112,34 @@ class DataContract:
         try:
             run.log_info("Testing data contract")
             data_contract = resolve.resolve_data_contract(
-                self._data_contract_file, self._data_contract_str, self._data_contract, self._schema_location
+                self._data_contract_file,
+                self._data_contract_str,
+                self._data_contract,
+                self._schema_location,
+                inline_definitions=self._inline_definitions,
+                inline_quality=self._inline_quality,
             )
 
-            if data_contract.models is None or len(data_contract.models) == 0:
-                raise DataContractException(
-                    type="lint",
-                    name="Check that data contract contains models",
-                    result="warning",
-                    reason="Models block is missing. Skip executing tests.",
-                    engine="datacontract",
-                )
-
-            if self._examples:
-                if data_contract.examples is None or len(data_contract.examples) == 0:
-                    raise DataContractException(
-                        type="lint",
-                        name="Check that data contract contains valid examples",
-                        result="warning",
-                        reason="Examples block is missing. Skip executing tests.",
-                        engine="datacontract",
-                    )
-            else:
-                check_that_datacontract_contains_valid_server_configuration(run, data_contract, self._server)
-
-            # TODO create directory only for examples
-            with tempfile.TemporaryDirectory(prefix="datacontract-cli") as tmp_dir:
-                if self._examples:
-                    server_name = "examples"
-                    server = self._get_examples_server(data_contract, run, tmp_dir)
-                else:
-                    server_name = list(data_contract.servers.keys())[0]
-                    server = data_contract.servers.get(server_name)
-
-                run.log_info(f"Running tests for data contract {data_contract.id} with server {server_name}")
-                run.dataContractId = data_contract.id
-                run.dataContractVersion = data_contract.info.version
-                run.dataProductId = server.dataProductId
-                run.outputPortId = server.outputPortId
-                run.server = server_name
-
-                # 5. check server is supported type
-                # 6. check server credentials are complete
-                if server.format == "json" and server.type != "kafka":
-                    check_jsonschema(run, data_contract, server)
-                check_soda_execute(run, data_contract, server, self._spark, tmp_dir)
+            execute_data_contract_test(data_contract, run, self._server, self._spark, self._duckdb_connection)
 
         except DataContractException as e:
             run.checks.append(
-                Check(type=e.type, result=e.result, name=e.name, reason=e.reason, engine=e.engine, details="")
+                Check(
+                    type=e.type,
+                    name=e.name,
+                    result=e.result,
+                    reason=e.reason,
+                    model=e.model,
+                    engine=e.engine,
+                    details="",
+                )
             )
             run.log_error(str(e))
         except Exception as e:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Test Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -222,16 +150,8 @@ class DataContract:
 
         run.finish()
 
-        if self._publish_url is not None:
-            try:
-                publish_datamesh_manager(run, self._publish_url)
-            except Exception:
-                run.log_error("Failed to publish to datamesh manager")
-        if self._publish_to_opentelemetry:
-            try:
-                publish_opentelemetry(run)
-            except Exception:
-                run.log_error("Failed to publish to opentelemetry")
+        if self._publish_url is not None or self._publish_test_results:
+            publish_test_results_to_datamesh_manager(run, self._publish_url, self._ssl_verification)
 
         return run
 
@@ -246,6 +166,24 @@ class DataContract:
 
         breaking_changes = list[BreakingChange]()
 
+        breaking_changes.extend(
+            info_breaking_changes(
+                old_info=old.info,
+                new_info=new.info,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
+
+        breaking_changes.extend(
+            terms_breaking_changes(
+                old_terms=old.terms,
+                new_terms=new.terms,
+                new_path=other._data_contract_file,
+                include_severities=include_severities,
+            )
+        )
+
         breaking_changes.extend(
             quality_breaking_changes(
                 old_quality=old.quality,
@@ -273,216 +211,141 @@ class DataContract:
             data_contract=self._data_contract,
             schema_location=self._schema_location,
             inline_definitions=self._inline_definitions,
+            inline_quality=self._inline_quality,
         )
 
-    def export(self, export_format, model: str = "all", rdf_base: str = None, sql_server_type: str = "auto") -> str:
-        data_contract = resolve.resolve_data_contract(
-            self._data_contract_file, self._data_contract_str, self._data_contract, inline_definitions=True
-        )
-        if export_format == "jsonschema":
-            if data_contract.models is None:
-                raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
-
-            model_names = list(data_contract.models.keys())
-
-            if model == "all":
-                if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(
-                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
-                    )
-
-                model_name, model_value = next(iter(data_contract.models.items()))
-                return to_jsonschema_json(model_name, model_value)
-            else:
-                model_name = model
-                model_value = data_contract.models.get(model_name)
-                if model_value is None:
-                    raise RuntimeError(
-                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
-                    )
-
-                return to_jsonschema_json(model_name, model_value)
-        if export_format == "sodacl":
-            return to_sodacl_yaml(data_contract)
-        if export_format == "dbt":
-            return to_dbt_models_yaml(data_contract)
-        if export_format == "dbt-sources":
-            return to_dbt_sources_yaml(data_contract, self._server)
-        if export_format == "dbt-staging-sql":
-            if data_contract.models is None:
-                raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
-
-            model_names = list(data_contract.models.keys())
-
-            if model == "all":
-                if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(
-                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
-                    )
-
-                model_name, model_value = next(iter(data_contract.models.items()))
-                return to_dbt_staging_sql(data_contract, model_name, model_value)
-            else:
-                model_name = model
-                model_value = data_contract.models.get(model_name)
-                if model_value is None:
-                    raise RuntimeError(
-                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
-                    )
-
-                return to_dbt_staging_sql(data_contract, model_name, model_value)
-        if export_format == "odcs":
-            return to_odcs_yaml(data_contract)
-        if export_format == "rdf":
-            return to_rdf_n3(data_contract, rdf_base)
-        if export_format == "protobuf":
-            return to_protobuf(data_contract)
-        if export_format == "avro":
-            if data_contract.models is None:
-                raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
-
-            model_names = list(data_contract.models.keys())
-
-            if model == "all":
-                if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(
-                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
-                    )
-
-                model_name, model_value = next(iter(data_contract.models.items()))
-                return to_avro_schema_json(model_name, model_value)
-            else:
-                model_name = model
-                model_value = data_contract.models.get(model_name)
-                if model_value is None:
-                    raise RuntimeError(
-                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
-                    )
-
-                return to_avro_schema_json(model_name, model_value)
-        if export_format == "avro-idl":
-            return to_avro_idl(data_contract)
-        if export_format == "terraform":
-            return to_terraform(data_contract)
-        if export_format == "sql":
-            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
-            return to_sql_ddl(data_contract, server_type=server_type)
-        if export_format == "sql-query":
-            if data_contract.models is None:
-                raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
-
-            server_type = self._determine_sql_server_type(data_contract, sql_server_type)
-
-            model_names = list(data_contract.models.keys())
-
-            if model == "all":
-                if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(
-                        f"Export to {export_format} is model specific. Specify the model via --model $MODEL_NAME. Available models: {model_names}"
-                    )
-
-                model_name, model_value = next(iter(data_contract.models.items()))
-                return to_sql_query(data_contract, model_name, model_value, server_type)
-            else:
-                model_name = model
-                model_value = data_contract.models.get(model_name)
-                if model_value is None:
-                    raise RuntimeError(
-                        f"Model {model_name} not found in the data contract. Available models: {model_names}"
-                    )
-
-                return to_sql_query(data_contract, model_name, model_value, server_type)
-
-        if export_format == "great-expectations":
-            if data_contract.models is None:
-                raise RuntimeError(f"Export to {export_format} requires models in the data contract.")
-
-            model_names = list(data_contract.models.keys())
-
-            if model == "all":
-                if len(data_contract.models.items()) != 1:
-                    raise RuntimeError(
-                        f"Export to {export_format} is model specific. Specify the model via --model "
-                        f"$MODEL_NAME. Available models: {model_names}"
-                    )
-
-                model_name, model_value = next(iter(data_contract.models.items()))
-                return to_great_expectations(data_contract, model_name)
-            else:
-                model_name = model
-                model_value = data_contract.models.get(model_name)
-                if model_value is None:
-                    raise RuntimeError(
-                        f"Model {model_name} not found in the data contract. " f"Available models: {model_names}"
-                    )
-
-                return to_great_expectations(data_contract, model_name)
-        if export_format == "pydantic-model":
-            return to_pydantic_model_str(data_contract)
-        if export_format == "html":
-            return to_html(data_contract)
-        else:
-            print(f"Export format {export_format} not supported.")
-            return ""
-
-    def _determine_sql_server_type(self, data_contract: DataContractSpecification, sql_server_type: str):
-        if sql_server_type == "auto":
-            if data_contract.servers is None or len(data_contract.servers) == 0:
-                raise RuntimeError("Export with server_type='auto' requires servers in the data contract.")
-
-            server_types = set([server.type for server in data_contract.servers.values()])
-            if "snowflake" in server_types:
-                return "snowflake"
-            elif "postgres" in server_types:
-                return "postgres"
-            elif "databricks" in server_types:
-                return "databricks"
-            else:
-                # default to snowflake dialect
-                return "snowflake"
+    def export(
+        self, export_format: ExportFormat, model: str = "all", sql_server_type: str = "auto", **kwargs
+    ) -> str | bytes:
+        if (
+            export_format == ExportFormat.html
+            or export_format == ExportFormat.mermaid
+            or export_format == ExportFormat.excel
+        ):
+            data_contract = resolve.resolve_data_contract_v2(
+                self._data_contract_file,
+                self._data_contract_str,
+                self._data_contract,
+                schema_location=self._schema_location,
+                inline_definitions=self._inline_definitions,
+                inline_quality=self._inline_quality,
+            )
+
+            return exporter_factory.create(export_format).export(
+                data_contract=data_contract,
+                model=model,
+                server=self._server,
+                sql_server_type=sql_server_type,
+                export_args=kwargs,
+            )
         else:
-            return sql_server_type
-
-    def _get_examples_server(self, data_contract, run, tmp_dir):
-        run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
-        format = "json"
-        for example in data_contract.examples:
-            format = example.type
-            p = f"{tmp_dir}/{example.model}.{format}"
-            run.log_info(f"Creating example file {p}")
-            with open(p, "w") as f:
-                content = ""
-                if format == "json" and isinstance(example.data, list):
-                    content = json.dumps(example.data)
-                elif format == "json" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "yaml" and isinstance(example.data, list):
-                    content = yaml.dump(example.data, allow_unicode=True)
-                elif format == "yaml" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "csv":
-                    content = example.data
-                logging.debug(f"Content of example file {p}: {content}")
-                f.write(content)
-        path = f"{tmp_dir}" + "/{model}." + format
-        delimiter = "array"
-        server = Server(
-            type="local",
-            path=path,
-            format=format,
-            delimiter=delimiter,
-        )
-        run.log_info(f"Using {server} for testing the examples")
-        return server
+            data_contract = resolve.resolve_data_contract(
+                self._data_contract_file,
+                self._data_contract_str,
+                self._data_contract,
+                schema_location=self._schema_location,
+                inline_definitions=self._inline_definitions,
+                inline_quality=self._inline_quality,
+            )
 
-    def import_from_source(self, format: str, source: str) -> DataContractSpecification:
-        data_contract_specification = DataContract.init()
+            return exporter_factory.create(export_format).export(
+                data_contract=data_contract,
+                model=model,
+                server=self._server,
+                sql_server_type=sql_server_type,
+                export_args=kwargs,
+            )
 
-        if format == "sql":
-            data_contract_specification = import_sql(data_contract_specification, format, source)
-        elif format == "avro":
-            data_contract_specification = import_avro(data_contract_specification, source)
+    @classmethod
+    def import_from_source(
+        cls,
+        format: str,
+        source: typing.Optional[str] = None,
+        template: typing.Optional[str] = None,
+        schema: typing.Optional[str] = None,
+        spec: Spec = Spec.datacontract_specification,
+        **kwargs,
+    ) -> DataContractSpecification | OpenDataContractStandard:
+        id = kwargs.get("id")
+        owner = kwargs.get("owner")
+
+        if spec == Spec.odcs or format == ImportFormat.excel:
+            data_contract_specification_initial = cls.init(template=template, schema=schema)
+
+            odcs_imported = importer_factory.create(format).import_source(
+                data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs
+            )
+
+            if isinstance(odcs_imported, DataContractSpecification):
+                # convert automatically
+                odcs_imported = to_odcs_v3(odcs_imported)
+
+            cls._overwrite_id_in_odcs(odcs_imported, id)
+            cls._overwrite_owner_in_odcs(odcs_imported, owner)
+
+            return odcs_imported
+        elif spec == Spec.datacontract_specification:
+            data_contract_specification_initial = cls.init(template=template, schema=schema)
+
+            data_contract_specification_imported = importer_factory.create(format).import_source(
+                data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs
+            )
+
+            if isinstance(data_contract_specification_imported, OpenDataContractStandard):
+                # convert automatically
+                data_contract_specification_imported = import_from_odcs(
+                    data_contract_specification_initial, data_contract_specification_imported
+                )
+
+            cls._overwrite_id_in_data_contract_specification(data_contract_specification_imported, id)
+            cls._overwrite_owner_in_data_contract_specification(data_contract_specification_imported, owner)
+
+            return data_contract_specification_imported
         else:
-            print(f"Import format {format} not supported.")
+            raise DataContractException(
+                type="general",
+                result=ResultEnum.error,
+                name="Import Data Contract",
+                reason=f"Unsupported data contract format: {spec}",
+                engine="datacontract",
+            )
 
-        return data_contract_specification
+    @staticmethod
+    def _overwrite_id_in_data_contract_specification(
+        data_contract_specification: DataContractSpecification, id: str | None
+    ):
+        if not id:
+            return
+
+        data_contract_specification.id = id
+
+    @staticmethod
+    def _overwrite_owner_in_data_contract_specification(
+        data_contract_specification: DataContractSpecification, owner: str | None
+    ):
+        if not owner:
+            return
+
+        if data_contract_specification.info is None:
+            data_contract_specification.info = Info()
+        data_contract_specification.info.owner = owner
+
+    @staticmethod
+    def _overwrite_owner_in_odcs(odcs: OpenDataContractStandard, owner: str | None):
+        if not owner:
+            return
+
+        if odcs.customProperties is None:
+            odcs.customProperties = []
+        for customProperty in odcs.customProperties:
+            if customProperty.name == "owner":
+                customProperty.value = owner
+                return
+        odcs.customProperties.append(CustomProperty(property="owner", value=owner))
+
+    @staticmethod
+    def _overwrite_id_in_odcs(odcs: OpenDataContractStandard, id: str | None):
+        if not id:
+            return
+
+        odcs.id = id