datacontract-cli 0.10.21__py3-none-any.whl → 0.10.23__py3-none-any.whl

datacontract/breaking/breaking.py

@@ -1,5 +1,5 @@
+from datacontract.breaking.breaking_change import BreakingChange, Location, Severity
 from datacontract.breaking.breaking_rules import BreakingRules
-from datacontract.model.breaking_change import BreakingChange, Location, Severity
 from datacontract.model.data_contract_specification import Contact, DeprecatedQuality, Field, Info, Model, Terms
 
 

datacontract/breaking/breaking_rules.py

@@ -1,4 +1,4 @@
-from datacontract.model.breaking_change import Severity
+from datacontract.breaking.breaking_change import Severity
 
 
 class BreakingRules:

datacontract/cli.py

@@ -5,9 +5,7 @@ from typing import Iterable, List, Optional
 
 import typer
 from click import Context
-from rich import box
 from rich.console import Console
-from rich.table import Table
 from typer.core import TyperGroup
 from typing_extensions import Annotated
 
@@ -19,6 +17,8 @@ from datacontract.integration.datamesh_manager import (
     publish_data_contract_to_datamesh_manager,
 )
 from datacontract.lint.resolve import resolve_data_contract_dict
+from datacontract.output.output_format import OutputFormat
+from datacontract.output.test_results_writer import write_test_result
 
 console = Console()
 
@@ -92,12 +92,19 @@ def lint(
         str,
         typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"),
     ] = None,
+    output: Annotated[
+        Path,
+        typer.Option(
+            help="Specify the file path where the test results should be written to (e.g., './test-results/TEST-datacontract.xml'). If no path is provided, the output will be printed to stdout."
+        ),
+    ] = None,
+    output_format: Annotated[OutputFormat, typer.Option(help="The target format for the test results.")] = None,
 ):
     """
     Validate that the datacontract.yaml is correctly formatted.
     """
     run = DataContract(data_contract_file=location, schema_location=schema).lint()
-    _handle_result(run)
+    write_test_result(run, console, output_format, output)
 
 
 @app.command()
@@ -119,11 +126,14 @@ def test(
             "servers (default)."
         ),
     ] = "all",
-    examples: Annotated[
-        bool,
-        typer.Option(help="Run the schema and quality tests on the example data within the data contract."),
-    ] = None,
     publish: Annotated[str, typer.Option(help="The url to publish the results after the test")] = None,
+    output: Annotated[
+        Path,
+        typer.Option(
+            help="Specify the file path where the test results should be written to (e.g., './test-results/TEST-datacontract.xml')."
+        ),
+    ] = None,
+    output_format: Annotated[OutputFormat, typer.Option(help="The target format for the test results.")] = None,
     logs: Annotated[bool, typer.Option(help="Print logs")] = False,
     ssl_verification: Annotated[
         bool,
@@ -141,12 +151,11 @@ def test(
         schema_location=schema,
         publish_url=publish,
         server=server,
-        examples=examples,
         ssl_verification=ssl_verification,
     ).test()
     if logs:
         _print_logs(run)
-    _handle_result(run)
+    write_test_result(run, console, output_format, output)
 
 
 @app.command()
@@ -219,7 +228,7 @@ def export(
     if output is None:
         console.print(result, markup=False, soft_wrap=True)
     else:
-        with output.open("w") as f:
+        with output.open(mode="w", encoding="utf-8") as f:
             f.write(result)
         console.print(f"Written result to {output}")
 
@@ -237,6 +246,10 @@ def import_(
         Optional[str],
         typer.Option(help="The path to the file or Glue Database that should be imported."),
     ] = None,
+    dialect: Annotated[
+        Optional[str],
+        typer.Option(help="The SQL dialect to use when importing SQL files, e.g., postgres, tsql, bigquery."),
+    ] = None,
     glue_table: Annotated[
         Optional[List[str]],
         typer.Option(
@@ -293,6 +306,7 @@ def import_(
         source=source,
         template=template,
         schema=schema,
+        dialect=dialect,
         glue_table=glue_table,
         bigquery_table=bigquery_table,
         bigquery_project=bigquery_project,
@@ -306,7 +320,7 @@ def import_(
     if output is None:
         console.print(result.to_yaml(), markup=False, soft_wrap=True)
     else:
-        with output.open("w") as f:
+        with output.open(mode="w", encoding="utf-8") as f:
             f.write(result.to_yaml())
         console.print(f"Written result to {output}")
 
@@ -467,77 +481,11 @@ def api(
     uvicorn.run(app="datacontract.api:app", port=port, host=host, reload=True, log_config=LOGGING_CONFIG)
 
 
-def _handle_result(run):
-    _print_table(run)
-    if run.result == "passed":
-        console.print(
-            f"🟢 data contract is valid. Run {len(run.checks)} checks. Took {(run.timestampEnd - run.timestampStart).total_seconds()} seconds."
-        )
-    elif run.result == "warning":
-        console.print("🟠 data contract has warnings. Found the following warnings:")
-        i = 1
-        for check in run.checks:
-            if check.result != "passed":
-                field = to_field(run, check)
-                if field:
-                    field = field + " "
-                else:
-                    field = ""
-                console.print(f"{i}) {field}{check.name}: {check.reason}")
-                i += 1
-    else:
-        console.print("🔴 data contract is invalid, found the following errors:")
-        i = 1
-        for check in run.checks:
-            if check.result != "passed":
-                field = to_field(run, check)
-                if field:
-                    field = field + " "
-                else:
-                    field = ""
-                console.print(f"{i}) {field}{check.name}: {check.reason}")
-                i += 1
-        raise typer.Exit(code=1)
-
-
-def _print_table(run):
-    table = Table(box=box.ROUNDED)
-    table.add_column("Result", no_wrap=True)
-    table.add_column("Check", max_width=100)
-    table.add_column("Field", max_width=32)
-    table.add_column("Details", max_width=50)
-    for check in sorted(run.checks, key=lambda c: (c.result or "", c.model or "", c.field or "")):
-        table.add_row(with_markup(check.result), check.name, to_field(run, check), check.reason)
-    console.print(table)
-
-
-def to_field(run, check):
-    models = [c.model for c in run.checks]
-    if len(set(models)) > 1:
-        if check.field is None:
-            return check.model
-        return check.model + "." + check.field
-    else:
-        return check.field
-
-
 def _print_logs(run):
     console.print("\nLogs:")
     for log in run.logs:
         console.print(log.timestamp.strftime("%y-%m-%d %H:%M:%S"), log.level.ljust(5), log.message)
 
 
-def with_markup(result):
-    if result == "passed":
-        return "[green]passed[/green]"
-    if result == "warning":
-        return "[yellow]warning[/yellow]"
-    if result == "failed":
-        return "[red]failed[/red]"
-    if result == "error":
-        return "[red]error[/red]"
-    return result
-
-
 if __name__ == "__main__":
     app()

datacontract/data_contract.py

@@ -1,10 +1,6 @@
-import json
 import logging
-import tempfile
 import typing
 
-import yaml
-
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
@@ -14,11 +10,8 @@ from datacontract.breaking.breaking import (
     quality_breaking_changes,
     terms_breaking_changes,
 )
-from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
-    check_that_datacontract_contains_valid_server_configuration,
-)
-from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
-from datacontract.engines.soda.check_soda_execute import check_soda_execute
+from datacontract.breaking.breaking_change import BreakingChange, BreakingChanges, Severity
+from datacontract.engines.data_contract_test import execute_data_contract_test
 from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
@@ -26,16 +19,14 @@ from datacontract.init.init_template import get_init_template
 from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
 from datacontract.lint import resolve
 from datacontract.lint.linters.description_linter import DescriptionLinter
-from datacontract.lint.linters.example_model_linter import ExampleModelLinter
 from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
 from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
 from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
-from datacontract.model.breaking_change import BreakingChange, BreakingChanges, Severity
-from datacontract.model.data_contract_specification import DataContractSpecification, Server
+from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Check, Run
+from datacontract.model.run import Check, ResultEnum, Run
 
 
 class DataContract:
@@ -46,7 +37,6 @@ class DataContract:
         data_contract: DataContractSpecification = None,
         schema_location: str = None,
         server: str = None,
-        examples: bool = False,
         publish_url: str = None,
         spark: "SparkSession" = None,
         inline_definitions: bool = True,
@@ -58,14 +48,12 @@ class DataContract:
         self._data_contract = data_contract
         self._schema_location = schema_location
         self._server = server
-        self._examples = examples
         self._publish_url = publish_url
         self._spark = spark
         self._inline_definitions = inline_definitions
         self._inline_quality = inline_quality
         self._ssl_verification = ssl_verification
         self.all_linters = {
-            ExampleModelLinter(),
             QualityUsesSchemaLinter(),
             FieldPatternLinter(),
             FieldReferenceLinter(),
@@ -96,7 +84,12 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
             run.checks.append(
-                Check(type="lint", result="passed", name="Data contract is syntactically valid", engine="datacontract")
+                Check(
+                    type="lint",
+                    result=ResultEnum.passed,
+                    name="Data contract is syntactically valid",
+                    engine="datacontract",
+                )
             )
             if enabled_linters == "none":
                 linters_to_check = set()
@@ -113,7 +106,7 @@ class DataContract:
                     run.checks.append(
                         Check(
                             type="general",
-                            result="error",
+                            result=ResultEnum.error,
                             name=f"Linter '{linter.name}'",
                             reason=str(e),
                             engine="datacontract",
@@ -130,7 +123,7 @@ class DataContract:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Check Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -153,54 +146,7 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
 
-            if data_contract.models is None or len(data_contract.models) == 0:
-                raise DataContractException(
-                    type="lint",
-                    name="Check that data contract contains models",
-                    result="warning",
-                    reason="Models block is missing. Skip executing tests.",
-                    engine="datacontract",
-                )
-
-            if self._examples:
-                if data_contract.examples is None or len(data_contract.examples) == 0:
-                    raise DataContractException(
-                        type="lint",
-                        name="Check that data contract contains valid examples",
-                        result="warning",
-                        reason="Examples block is missing. Skip executing tests.",
-                        engine="datacontract",
-                    )
-            else:
-                check_that_datacontract_contains_valid_server_configuration(run, data_contract, self._server)
-
-            # TODO create directory only for examples
-            with tempfile.TemporaryDirectory(prefix="datacontract-cli") as tmp_dir:
-                if self._examples:
-                    server_name = "examples"
-                    server = self._get_examples_server(data_contract, run, tmp_dir)
-                elif self._server:
-                    server_name = self._server
-                    server = data_contract.servers.get(server_name)
-                else:
-                    server_name = list(data_contract.servers.keys())[0]
-                    server = data_contract.servers.get(server_name)
-
-                run.log_info(f"Running tests for data contract {data_contract.id} with server {server_name}")
-                run.dataContractId = data_contract.id
-                run.dataContractVersion = data_contract.info.version
-                run.dataProductId = server.dataProductId
-                run.outputPortId = server.outputPortId
-                run.server = server_name
-
-                # TODO check server is supported type for nicer error messages
-
-                # TODO check server credentials are complete for nicer error messages
-
-                if server.format == "json" and server.type != "kafka":
-                    check_jsonschema(run, data_contract, server)
-
-                check_soda_execute(run, data_contract, server, self._spark, tmp_dir)
+            execute_data_contract_test(data_contract, run, self._server, self._spark)
 
         except DataContractException as e:
             run.checks.append(
@@ -219,7 +165,7 @@ class DataContract:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Test Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -235,38 +181,6 @@ class DataContract:
 
         return run
 
-    def _get_examples_server(self, data_contract, run, tmp_dir):
-        run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
-        format = "json"
-        for example in data_contract.examples:
-            format = example.type
-            p = f"{tmp_dir}/{example.model}.{format}"
-            run.log_info(f"Creating example file {p}")
-            with open(p, "w") as f:
-                content = ""
-                if format == "json" and isinstance(example.data, list):
-                    content = json.dumps(example.data)
-                elif format == "json" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "yaml" and isinstance(example.data, list):
-                    content = yaml.dump(example.data, allow_unicode=True)
-                elif format == "yaml" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "csv":
-                    content = example.data
-                logging.debug(f"Content of example file {p}: {content}")
-                f.write(content)
-        path = f"{tmp_dir}" + "/{model}." + format
-        delimiter = "array"
-        server = Server(
-            type="local",
-            path=path,
-            format=format,
-            delimiter=delimiter,
-        )
-        run.log_info(f"Using {server} for testing the examples")
-        return server
-
     def breaking(self, other: "DataContract") -> BreakingChanges:
         return self.changelog(other, include_severities=[Severity.ERROR, Severity.WARNING])