datacontract-cli 0.10.20__py3-none-any.whl → 0.10.22__py3-none-any.whl

datacontract/{web.py → api.py}

@@ -1,8 +1,11 @@
+import logging
+import os
 from typing import Annotated, Optional
 
 import typer
-from fastapi import Body, FastAPI, Query
+from fastapi import Body, Depends, FastAPI, HTTPException, Query, status
 from fastapi.responses import PlainTextResponse
+from fastapi.security.api_key import APIKeyHeader
 
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.model.run import Run
@@ -55,8 +58,8 @@ models:
 
 app = FastAPI(
     docs_url="/",
-    title="Data Contract API",
-    summary="API to execute Data Contract CLI operations.",
+    title="Data Contract CLI API",
+    summary="You can use the API to test, export, and lint your data contracts.",
     license_info={
         "name": "MIT License",
         "identifier": "MIT",
@@ -87,6 +90,32 @@ app = FastAPI(
     ],
 )
 
+api_key_header = APIKeyHeader(
+    name="x-api-key",
+    auto_error=False,  # this makes authentication optional
+)
+
+
+def check_api_key(api_key_header: str | None):
+    correct_api_key = os.getenv("DATACONTRACT_CLI_API_KEY")
+    if correct_api_key is None or correct_api_key == "":
+        logging.info("Environment variable DATACONTRACT_CLI_API_KEY is not set. Skip API key check.")
+        return
+    if api_key_header is None or api_key_header == "":
+        logging.info("The API key is missing.")
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing API key. Use Header 'x-api-key' to provide the API key.",
+        )
+    if api_key_header != correct_api_key:
+        logging.info("The provided API key is not correct.")
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="The provided API key is not correct.",
+        )
+    logging.info("Request authenticated with API key.")
+    pass
+
 
 @app.post(
     "/test",
@@ -98,6 +127,25 @@ app = FastAPI(
               Credentials must be provided via environment variables when running the web server.
               POST the data contract YAML as payload.
             """,
+    responses={
+        401: {
+            "description": "Unauthorized (when an environment variable DATACONTRACT_CLI_API_KEY is configured).",
+            "content": {
+                "application/json": {
+                    "examples": {
+                        "api_key_missing": {
+                            "summary": "API key Missing",
+                            "value": {"detail": "Missing API key. Use Header 'x-api-key' to provide the API key."},
+                        },
+                        "api_key_wrong": {
+                            "summary": "API key Wrong",
+                            "value": {"detail": "The provided API key is not correct."},
+                        },
+                    }
+                }
+            },
+        },
+    },
     response_model_exclude_none=True,
     response_model_exclude_unset=True,
 )
@@ -110,6 +158,7 @@ async def test(
             examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
         ),
     ],
+    api_key: Annotated[str | None, Depends(api_key_header)] = None,
     server: Annotated[
         str | None,
         Query(
@@ -118,6 +167,9 @@ async def test(
         ),
     ] = None,
 ) -> Run:
+    check_api_key(api_key)
+    logging.info("Testing data contract...")
+    logging.info(body)
     return DataContract(data_contract_str=body, server=server).test()
 
 

datacontract/breaking/breaking.py

@@ -1,5 +1,5 @@
+from datacontract.breaking.breaking_change import BreakingChange, Location, Severity
 from datacontract.breaking.breaking_rules import BreakingRules
-from datacontract.model.breaking_change import BreakingChange, Location, Severity
 from datacontract.model.data_contract_specification import Contact, DeprecatedQuality, Field, Info, Model, Terms
 
 

datacontract/breaking/breaking_rules.py

@@ -1,4 +1,4 @@
-from datacontract.model.breaking_change import Severity
+from datacontract.breaking.breaking_change import Severity
 
 
 class BreakingRules:

datacontract/cli.py

@@ -119,10 +119,6 @@ def test(
             "servers (default)."
         ),
     ] = "all",
-    examples: Annotated[
-        bool,
-        typer.Option(help="Run the schema and quality tests on the example data within the data contract."),
-    ] = None,
     publish: Annotated[str, typer.Option(help="The url to publish the results after the test")] = None,
     logs: Annotated[bool, typer.Option(help="Print logs")] = False,
     ssl_verification: Annotated[
@@ -141,7 +137,6 @@ def test(
         schema_location=schema,
         publish_url=publish,
         server=server,
-        examples=examples,
         ssl_verification=ssl_verification,
     ).test()
     if logs:
@@ -196,6 +191,11 @@ def export(
         Optional[str],
         typer.Option(help="[engine] The engine used for great expection run."),
     ] = None,
+    # TODO: this should be a subcommand
+    template: Annotated[
+        Optional[Path],
+        typer.Option(help="[custom] The file path of Jinja template."),
+    ] = None,
 ):
     """
     Convert data contract to a specific format. Saves to file specified by `output` option if present, otherwise prints to stdout.
@@ -208,6 +208,7 @@ def export(
         rdf_base=rdf_base,
         sql_server_type=sql_server_type,
         engine=engine,
+        template=template,
     )
     # Don't interpret console markup in output.
     if output is None:
@@ -231,6 +232,10 @@ def import_(
         Optional[str],
         typer.Option(help="The path to the file or Glue Database that should be imported."),
     ] = None,
+    dialect: Annotated[
+        Optional[str],
+        typer.Option(help="The SQL dialect to use when importing SQL files, e.g., postgres, tsql, bigquery."),
+    ] = None,
     glue_table: Annotated[
         Optional[List[str]],
         typer.Option(
@@ -287,6 +292,7 @@ def import_(
         source=source,
         template=template,
         schema=schema,
+        dialect=dialect,
         glue_table=glue_table,
         bigquery_table=bigquery_table,
         bigquery_project=bigquery_project,
@@ -344,7 +350,7 @@ def catalog(
     ] = None,
 ):
     """
-    Create an html catalog of data contracts.
+    Create a html catalog of data contracts.
     """
     path = Path(output)
     path.mkdir(parents=True, exist_ok=True)
@@ -433,16 +439,32 @@ def diff(
 
 
 @app.command()
-def serve(
+def api(
     port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4242,
-    host: Annotated[str, typer.Option(help="Bind socket to this host.")] = "127.0.0.1",
+    host: Annotated[
+        str, typer.Option(help="Bind socket to this host. Hint: For running in docker, set it to 0.0.0.0")
+    ] = "127.0.0.1",
 ):
     """
-    Start the datacontract web server.
+    Start the datacontract CLI as server application with REST API.
+
+    The OpenAPI documentation as Swagger UI is available on http://localhost:4242.
+    You can execute the commands directly from the Swagger UI.
+
+    To protect the API, you can set the environment variable DATACONTRACT_CLI_API_KEY to a secret API key.
+    To authenticate, requests must include the header 'x-api-key' with the correct API key.
+    This is highly recommended, as data contract tests may be subject to SQL injections or leak sensitive information.
+
+    To connect to servers (such as a Snowflake data source), set the credentials as environment variables as documented in
+    https://cli.datacontract.com/#test
     """
     import uvicorn
+    from uvicorn.config import LOGGING_CONFIG
+
+    log_config = LOGGING_CONFIG
+    log_config["root"] = {"level": "INFO"}
 
-    uvicorn.run("datacontract.web:app", port=port, host=host, reload=True)
+    uvicorn.run(app="datacontract.api:app", port=port, host=host, reload=True, log_config=LOGGING_CONFIG)
 
 
 def _handle_result(run):

datacontract/data_contract.py

@@ -1,10 +1,6 @@
-import json
 import logging
-import tempfile
 import typing
 
-import yaml
-
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
 
@@ -14,11 +10,8 @@ from datacontract.breaking.breaking import (
     quality_breaking_changes,
     terms_breaking_changes,
 )
-from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
-    check_that_datacontract_contains_valid_server_configuration,
-)
-from datacontract.engines.fastjsonschema.check_jsonschema import check_jsonschema
-from datacontract.engines.soda.check_soda_execute import check_soda_execute
+from datacontract.breaking.breaking_change import BreakingChange, BreakingChanges, Severity
+from datacontract.engines.data_contract_test import execute_data_contract_test
 from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
@@ -26,16 +19,14 @@ from datacontract.init.init_template import get_init_template
 from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
 from datacontract.lint import resolve
 from datacontract.lint.linters.description_linter import DescriptionLinter
-from datacontract.lint.linters.example_model_linter import ExampleModelLinter
 from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter
 from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter
 from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter
 from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter
 from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter
-from datacontract.model.breaking_change import BreakingChange, BreakingChanges, Severity
-from datacontract.model.data_contract_specification import DataContractSpecification, Server
+from datacontract.model.data_contract_specification import DataContractSpecification
 from datacontract.model.exceptions import DataContractException
-from datacontract.model.run import Check, Run
+from datacontract.model.run import Check, ResultEnum, Run
 
 
 class DataContract:
@@ -46,7 +37,6 @@ class DataContract:
         data_contract: DataContractSpecification = None,
         schema_location: str = None,
         server: str = None,
-        examples: bool = False,
         publish_url: str = None,
         spark: "SparkSession" = None,
         inline_definitions: bool = True,
@@ -58,14 +48,12 @@ class DataContract:
         self._data_contract = data_contract
         self._schema_location = schema_location
         self._server = server
-        self._examples = examples
         self._publish_url = publish_url
         self._spark = spark
         self._inline_definitions = inline_definitions
         self._inline_quality = inline_quality
         self._ssl_verification = ssl_verification
         self.all_linters = {
-            ExampleModelLinter(),
             QualityUsesSchemaLinter(),
             FieldPatternLinter(),
             FieldReferenceLinter(),
@@ -96,7 +84,12 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
             run.checks.append(
-                Check(type="lint", result="passed", name="Data contract is syntactically valid", engine="datacontract")
+                Check(
+                    type="lint",
+                    result=ResultEnum.passed,
+                    name="Data contract is syntactically valid",
+                    engine="datacontract",
+                )
             )
             if enabled_linters == "none":
                 linters_to_check = set()
@@ -113,7 +106,7 @@ class DataContract:
                     run.checks.append(
                         Check(
                             type="general",
-                            result="error",
+                            result=ResultEnum.error,
                             name=f"Linter '{linter.name}'",
                             reason=str(e),
                             engine="datacontract",
@@ -130,7 +123,7 @@ class DataContract:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Check Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -153,54 +146,7 @@ class DataContract:
                 inline_quality=self._inline_quality,
             )
 
-            if data_contract.models is None or len(data_contract.models) == 0:
-                raise DataContractException(
-                    type="lint",
-                    name="Check that data contract contains models",
-                    result="warning",
-                    reason="Models block is missing. Skip executing tests.",
-                    engine="datacontract",
-                )
-
-            if self._examples:
-                if data_contract.examples is None or len(data_contract.examples) == 0:
-                    raise DataContractException(
-                        type="lint",
-                        name="Check that data contract contains valid examples",
-                        result="warning",
-                        reason="Examples block is missing. Skip executing tests.",
-                        engine="datacontract",
-                    )
-            else:
-                check_that_datacontract_contains_valid_server_configuration(run, data_contract, self._server)
-
-            # TODO create directory only for examples
-            with tempfile.TemporaryDirectory(prefix="datacontract-cli") as tmp_dir:
-                if self._examples:
-                    server_name = "examples"
-                    server = self._get_examples_server(data_contract, run, tmp_dir)
-                elif self._server:
-                    server_name = self._server
-                    server = data_contract.servers.get(server_name)
-                else:
-                    server_name = list(data_contract.servers.keys())[0]
-                    server = data_contract.servers.get(server_name)
-
-                run.log_info(f"Running tests for data contract {data_contract.id} with server {server_name}")
-                run.dataContractId = data_contract.id
-                run.dataContractVersion = data_contract.info.version
-                run.dataProductId = server.dataProductId
-                run.outputPortId = server.outputPortId
-                run.server = server_name
-
-                # TODO check server is supported type for nicer error messages
-
-                # TODO check server credentials are complete for nicer error messages
-
-                if server.format == "json" and server.type != "kafka":
-                    check_jsonschema(run, data_contract, server)
-
-                check_soda_execute(run, data_contract, server, self._spark, tmp_dir)
+            execute_data_contract_test(data_contract, run, self._server, self._spark)
 
         except DataContractException as e:
             run.checks.append(
@@ -219,7 +165,7 @@ class DataContract:
             run.checks.append(
                 Check(
                     type="general",
-                    result="error",
+                    result=ResultEnum.error,
                     name="Test Data Contract",
                     reason=str(e),
                     engine="datacontract",
@@ -235,38 +181,6 @@ class DataContract:
 
         return run
 
-    def _get_examples_server(self, data_contract, run, tmp_dir):
-        run.log_info(f"Copying examples to files in temporary directory {tmp_dir}")
-        format = "json"
-        for example in data_contract.examples:
-            format = example.type
-            p = f"{tmp_dir}/{example.model}.{format}"
-            run.log_info(f"Creating example file {p}")
-            with open(p, "w") as f:
-                content = ""
-                if format == "json" and isinstance(example.data, list):
-                    content = json.dumps(example.data)
-                elif format == "json" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "yaml" and isinstance(example.data, list):
-                    content = yaml.dump(example.data, allow_unicode=True)
-                elif format == "yaml" and isinstance(example.data, str):
-                    content = example.data
-                elif format == "csv":
-                    content = example.data
-                logging.debug(f"Content of example file {p}: {content}")
-                f.write(content)
-        path = f"{tmp_dir}" + "/{model}." + format
-        delimiter = "array"
-        server = Server(
-            type="local",
-            path=path,
-            format=format,
-            delimiter=delimiter,
-        )
-        run.log_info(f"Using {server} for testing the examples")
-        return server
-
     def breaking(self, other: "DataContract") -> BreakingChanges:
         return self.changelog(other, include_severities=[Severity.ERROR, Severity.WARNING])