datacontract-cli 0.10.21__py3-none-any.whl → 0.10.22__py3-none-any.whl

datacontract/breaking/breaking.py

@@ -1,5 +1,5 @@
+from datacontract.breaking.breaking_change import BreakingChange, Location, Severity
 from datacontract.breaking.breaking_rules import BreakingRules
-from datacontract.model.breaking_change import BreakingChange, Location, Severity
 from datacontract.model.data_contract_specification import Contact, DeprecatedQuality, Field, Info, Model, Terms
 
 

datacontract/breaking/breaking_rules.py

@@ -1,4 +1,4 @@
-from datacontract.model.breaking_change import Severity
+from datacontract.breaking.breaking_change import Severity
 
 
 class BreakingRules:

datacontract/cli.py

@@ -119,10 +119,6 @@ def test(
             "servers (default)."
         ),
     ] = "all",
-    examples: Annotated[
-        bool,
-        typer.Option(help="Run the schema and quality tests on the example data within the data contract."),
-    ] = None,
     publish: Annotated[str, typer.Option(help="The url to publish the results after the test")] = None,
     logs: Annotated[bool, typer.Option(help="Print logs")] = False,
     ssl_verification: Annotated[
@@ -141,7 +137,6 @@ def test(
         schema_location=schema,
         publish_url=publish,
         server=server,
-        examples=examples,
         ssl_verification=ssl_verification,
     ).test()
     if logs:
@@ -237,6 +232,10 @@ def import_(
         Optional[str],
         typer.Option(help="The path to the file or Glue Database that should be imported."),
     ] = None,
+    dialect: Annotated[
+        Optional[str],
+        typer.Option(help="The SQL dialect to use when importing SQL files, e.g., postgres, tsql, bigquery."),
+    ] = None,
     glue_table: Annotated[
         Optional[List[str]],
         typer.Option(
@@ -293,6 +292,7 @@ def import_(
         source=source,
         template=template,
         schema=schema,
+        dialect=dialect,
         glue_table=glue_table,
         bigquery_table=bigquery_table,
         bigquery_project=bigquery_project,

datacontract/data_contract.py

@@ -1,10 +1,6 @@
-import json
 import logging
-import tempfile
 import typing
 
-import yaml
-
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
@@ -14,11 +10,8 @@ from datacontract.breaking.breaking import (
     quality_breaking_changes,
     terms_breaking_changes,
 )
-from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
-    check_that_datacontract_contains_valid_server_configuration,
-)
-from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
-from datacontract.engines.soda.check_soda_execute import check_soda_execute
+from datacontract.breaking.breaking_change import BreakingChange, BreakingChanges, Severity
+from datacontract.engines.data_contract_test import execute_data_contract_test
 from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
@@ -26,16 +19,14 @@ from datacontract.init.init_template import get_init_template
 from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
 from datacontract.lint import resolve
 from datacontract.lint.linters.description_linter import DescriptionLinter
-from datacontract.lint.linters.example_model_linter import ExampleModelLinter
 from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
 from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
 from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
-from datacontract.model.breaking_change import BreakingChange, BreakingChanges, Severity
-from datacontract.model.data_contract_specification import DataContractSpecification, Server
+from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Check, Run
+from datacontract.model.run import Check, ResultEnum, Run
 
 
 class DataContract:
@@ -46,7 +37,6 @@ class DataContract:
         data_contract: DataContractSpecification = None,
         schema_location: str = None,
         server: str = None,
-        examples: bool = False,
         publish_url: str = None,
         spark: "SparkSession" = None,
         inline_definitions: bool = True,
@@ -58,14 +48,12 @@ class DataContract:
         self._data_contract = data_contract
         self._schema_location = schema_location
         self._server = server
-        self._examples = examples
         self._publish_url = publish_url
         self._spark = spark
         self._inline_definitions = inline_definitions
         self._inline_quality = inline_quality
         self._ssl_verification = ssl_verification
         self.all_linters = {
-            ExampleModelLinter(),
             QualityUsesSchemaLinter(),
             FieldPatternLinter(),
             FieldReferenceLinter(),
@@ -96,7 +84,12 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
             run.checks.append(
-                Check(type="lint", result="passed", name="Data contract is syntactically valid", engine="datacontract")
+                Check(
+                    type="lint",
+                    result=ResultEnum.passed,
+                    name="Data contract is syntactically valid",
+                    engine="datacontract",
+                )
             )
             if enabled_linters == "none":
                 linters_to_check = set()
@@ -113,7 +106,7 @@ class DataContract:
                     run.checks.append(
                         Check(
                             type="general",
-                            result="error",
+                            result=ResultEnum.error,
                             name=f"Linter '{linter.name}'",
                             reason=str(e),
                             engine="datacontract",
@@ -130,7 +123,7 @@ class DataContract:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Check Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -153,54 +146,7 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
 
-            if data_contract.models is None or len(data_contract.models) == 0:
-                raise DataContractException(
-                    type="lint",
-                    name="Check that data contract contains models",
-                    result="warning",
-                    reason="Models block is missing. Skip executing tests.",
-                    engine="datacontract",
-                )
-
-            if self._examples:
-                if data_contract.examples is None or len(data_contract.examples) == 0:
-                    raise DataContractException(
-                        type="lint",
-                        name="Check that data contract contains valid examples",
-                        result="warning",
-                        reason="Examples block is missing. Skip executing tests.",
-                        engine="datacontract",
-                    )
-            else:
-                check_that_datacontract_contains_valid_server_configuration(run, data_contract, self._server)
-
-            # TODO create directory only for examples
-            with tempfile.TemporaryDirectory(prefix="datacontract-cli") as tmp_dir:
-                if self._examples:
-                    server_name = "examples"
-                    server = self._get_examples_server(data_contract, run, tmp_dir)
-                elif self._server:
-                    server_name = self._server
-                    server = data_contract.servers.get(server_name)
-                else:
-                    server_name = list(data_contract.servers.keys())[0]
-                    server = data_contract.servers.get(server_name)
-
-                run.log_info(f"Running tests for data contract {data_contract.id} with server {server_name}")
-                run.dataContractId = data_contract.id
-                run.dataContractVersion = data_contract.info.version
-                run.dataProductId = server.dataProductId
-                run.outputPortId = server.outputPortId
-                run.server = server_name
-
-                # TODO check server is supported type for nicer error messages
-
-                # TODO check server credentials are complete for nicer error messages
-
-                if server.format == "json" and server.type != "kafka":
-                    check_jsonschema(run, data_contract, server)
-
-                check_soda_execute(run, data_contract, server, self._spark, tmp_dir)
+            execute_data_contract_test(data_contract, run, self._server, self._spark)
 
         except DataContractException as e:
             run.checks.append(
@@ -219,7 +165,7 @@ class DataContract:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Test Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -235,38 +181,6 @@ class DataContract:
 
         return run
 
-    def _get_examples_server(self, data_contract, run, tmp_dir):
-        run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
-        format = "json"
-        for example in data_contract.examples:
-            format = example.type
-            p = f"{tmp_dir}/{example.model}.{format}"
-            run.log_info(f"Creating example file {p}")
-            with open(p, "w") as f:
-                content = ""
-                if format == "json" and isinstance(example.data, list):
-                    content = json.dumps(example.data)
-                elif format == "json" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "yaml" and isinstance(example.data, list):
-                    content = yaml.dump(example.data, allow_unicode=True)
-                elif format == "yaml" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "csv":
-                    content = example.data
-                logging.debug(f"Content of example file {p}: {content}")
-                f.write(content)
-        path = f"{tmp_dir}" + "/{model}." + format
-        delimiter = "array"
-        server = Server(
-            type="local",
-            path=path,
-            format=format,
-            delimiter=delimiter,
-        )
-        run.log_info(f"Using {server} for testing the examples")
-        return server
-
     def breaking(self, other: "DataContract") -> BreakingChanges:
         return self.changelog(other, include_severities=[Severity.ERROR, Severity.WARNING])