datacontract-cli 0.10.19__py3-none-any.whl → 0.10.21__py3-none-any.whl

datacontract/api.py

@@ -0,0 +1,253 @@
+import logging
+import os
+from typing import Annotated, Optional
+
+import typer
+from fastapi import Body, Depends, FastAPI, HTTPException, Query, status
+from fastapi.responses import PlainTextResponse
+from fastapi.security.api_key import APIKeyHeader
+
+from datacontract.data_contract import DataContract, ExportFormat
+from datacontract.model.run import Run
+
+DATA_CONTRACT_EXAMPLE_PAYLOAD = """dataContractSpecification: 1.1.0
+id: urn:datacontract:checkout:orders-latest
+info:
+  title: Orders Latest
+  version: 2.0.0
+  owner: Sales Team
+servers:
+  production:
+    type: s3
+    location: s3://datacontract-example-orders-latest/v2/{model}/*.json
+    format: json
+    delimiter: new_line
+models:
+  orders:
+    description: One record per order. Includes cancelled and deleted orders.
+    type: table
+    fields:
+      order_id:
+        type: string
+        primaryKey: true
+      order_timestamp:
+        description: The business timestamp in UTC when the order was successfully registered in the source system and the payment was successful.
+        type: timestamp
+        required: true
+        examples:
+          - "2024-09-09T08:30:00Z"
+      order_total:
+        description: Total amount the smallest monetary unit (e.g., cents).
+        type: long
+        required: true
+        examples:
+          - 9999
+        quality:
+          - type: sql
+            description: 95% of all order total values are expected to be between 10 and 499 EUR.
+            query: |
+              SELECT quantile_cont(order_total, 0.95) AS percentile_95
+              FROM orders
+            mustBeBetween: [1000, 99900]
+      customer_id:
+        description: Unique identifier for the customer.
+        type: text
+        minLength: 10
+        maxLength: 20
+"""
+
+app = FastAPI(
+    docs_url="/",
+    title="Data Contract CLI API",
+    summary="You can use the API to test, export, and lint your data contracts.",
+    license_info={
+        "name": "MIT License",
+        "identifier": "MIT",
+    },
+    contact={"name": "Data Contract CLI", "url": "https://cli.datacontract.com/"},
+    openapi_tags=[
+        {
+            "name": "test",
+            "externalDocs": {
+                "description": "Documentation",
+                "url": "https://cli.datacontract.com/#test",
+            },
+        },
+        {
+            "name": "lint",
+            "externalDocs": {
+                "description": "Documentation",
+                "url": "https://cli.datacontract.com/#lint",
+            },
+        },
+        {
+            "name": "export",
+            "externalDocs": {
+                "description": "Documentation",
+                "url": "https://cli.datacontract.com/#export",
+            },
+        },
+    ],
+)
+
+api_key_header = APIKeyHeader(
+    name="x-api-key",
+    auto_error=False,  # this makes authentication optional
+)
+
+
+def check_api_key(api_key_header: str | None):
+    correct_api_key = os.getenv("DATACONTRACT_CLI_API_KEY")
+    if correct_api_key is None or correct_api_key == "":
+        logging.info("Environment variable DATACONTRACT_CLI_API_KEY is not set. Skip API key check.")
+        return
+    if api_key_header is None or api_key_header == "":
+        logging.info("The API key is missing.")
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing API key. Use Header 'x-api-key' to provide the API key.",
+        )
+    if api_key_header != correct_api_key:
+        logging.info("The provided API key is not correct.")
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="The provided API key is not correct.",
+        )
+    logging.info("Request authenticated with API key.")
+    pass
+
+
+@app.post(
+    "/test",
+    tags=["test"],
+    summary="Run data contract tests",
+    description="""
+              Run schema and quality tests. Data Contract CLI connects to the data sources configured in the server section.
+              This usually requires credentials to access the data sources.
+              Credentials must be provided via environment variables when running the web server.
+              POST the data contract YAML as payload.
+            """,
+    responses={
+        401: {
+            "description": "Unauthorized (when an environment variable DATACONTRACT_CLI_API_KEY is configured).",
+            "content": {
+                "application/json": {
+                    "examples": {
+                        "api_key_missing": {
+                            "summary": "API key Missing",
+                            "value": {"detail": "Missing API key. Use Header 'x-api-key' to provide the API key."},
+                        },
+                        "api_key_wrong": {
+                            "summary": "API key Wrong",
+                            "value": {"detail": "The provided API key is not correct."},
+                        },
+                    }
+                }
+            },
+        },
+    },
+    response_model_exclude_none=True,
+    response_model_exclude_unset=True,
+)
+async def test(
+    body: Annotated[
+        str,
+        Body(
+            title="Data Contract YAML",
+            media_type="application/yaml",
+            examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
+        ),
+    ],
+    api_key: Annotated[str | None, Depends(api_key_header)] = None,
+    server: Annotated[
+        str | None,
+        Query(
+            example="production",
+            description="The server name to test. Optional, if there is only one server.",
+        ),
+    ] = None,
+) -> Run:
+    check_api_key(api_key)
+    logging.info("Testing data contract...")
+    logging.info(body)
+    return DataContract(data_contract_str=body, server=server).test()
+
+
+@app.post(
+    "/lint",
+    tags=["lint"],
+    summary="Validate that the datacontract.yaml is correctly formatted.",
+    description="""Validate that the datacontract.yaml is correctly formatted.""",
+)
+async def lint(
+    body: Annotated[
+        str,
+        Body(
+            title="Data Contract YAML",
+            media_type="application/yaml",
+            examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
+        ),
+    ],
+    schema: Annotated[
+        str | None,
+        Query(
+            example="https://datacontract.com/datacontract.schema.json",
+            description="The schema to use for validation. This must be a URL.",
+        ),
+    ] = None,
+):
+    data_contract = DataContract(data_contract_str=body, schema_location=schema)
+    lint_result = data_contract.lint()
+    return {"result": lint_result.result, "checks": lint_result.checks}
+
+
+@app.post(
+    "/export",
+    tags=["export"],
+    summary="Convert data contract to a specific format.",
+    response_class=PlainTextResponse,
+)
+def export(
+    body: Annotated[
+        str,
+        Body(
+            title="Data Contract YAML",
+            media_type="application/yaml",
+            examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
+        ),
+    ],
+    format: Annotated[ExportFormat, typer.Option(help="The export format.")],
+    server: Annotated[
+        str | None,
+        Query(
+            example="production",
+            description="The server name to export. Optional, if there is only one server.",
+        ),
+    ] = None,
+    model: Annotated[
+        str | None,
+        Query(
+            description="Use the key of the model in the data contract yaml file "
+            "to refer to a model, e.g., `orders`, or `all` for all "
+            "models (default).",
+        ),
+    ] = "all",
+    rdf_base: Annotated[
+        Optional[str],
+        typer.Option(help="[rdf] The base URI used to generate the RDF graph.", rich_help_panel="RDF Options"),
+    ] = None,
+    sql_server_type: Annotated[
+        Optional[str],
+        Query(
+            description="[sql] The server type to determine the sql dialect. By default, it uses 'auto' to automatically detect the sql dialect via the specified servers in the data contract.",
+        ),
+    ] = None,
+):
+    result = DataContract(data_contract_str=body, server=server).export(
+        export_format=format,
+        model=model,
+        rdf_base=rdf_base,
+        sql_server_type=sql_server_type,
+    )
+
+    return result

datacontract/cli.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import Iterable, List, Optional
 
 import typer
-import uvicorn
 from click import Context
 from rich import box
 from rich.console import Console
@@ -12,7 +11,6 @@ from rich.table import Table
 from typer.core import TyperGroup
 from typing_extensions import Annotated
 
-from datacontract import web
 from datacontract.catalog.catalog import create_data_contract_html, create_index_html
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.imports.importer import ImportFormat
@@ -198,6 +196,11 @@ def export(
         Optional[str],
         typer.Option(help="[engine] The engine used for great expection run."),
     ] = None,
+    # TODO: this should be a subcommand
+    template: Annotated[
+        Optional[Path],
+        typer.Option(help="[custom] The file path of Jinja template."),
+    ] = None,
 ):
     """
     Convert data contract to a specific format. Saves to file specified by `output` option if present, otherwise prints to stdout.
@@ -210,6 +213,7 @@ def export(
         rdf_base=rdf_base,
         sql_server_type=sql_server_type,
         engine=engine,
+        template=template,
     )
     # Don't interpret console markup in output.
     if output is None:
@@ -346,7 +350,7 @@ def catalog(
     ] = None,
 ):
     """
-    Create an html catalog of data contracts.
+    Create a html catalog of data contracts.
     """
     path = Path(output)
     path.mkdir(parents=True, exist_ok=True)
@@ -435,15 +439,32 @@ def diff(
 
 
 @app.command()
-def serve(
+def api(
     port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4242,
-    host: Annotated[str, typer.Option(help="Bind socket to this host.")] = "127.0.0.1",
+    host: Annotated[
+        str, typer.Option(help="Bind socket to this host. Hint: For running in docker, set it to 0.0.0.0")
+    ] = "127.0.0.1",
 ):
     """
-    Start the datacontract web server.
+    Start the datacontract CLI as server application with REST API.
+
+    The OpenAPI documentation as Swagger UI is available on http://localhost:4242.
+    You can execute the commands directly from the Swagger UI.
+
+    To protect the API, you can set the environment variable DATACONTRACT_CLI_API_KEY to a secret API key.
+    To authenticate, requests must include the header 'x-api-key' with the correct API key.
+    This is highly recommended, as data contract tests may be subject to SQL injections or leak sensitive information.
+
+    To connect to servers (such as a Snowflake data source), set the credentials as environment variables as documented in
+    https://cli.datacontract.com/#test
     """
+    import uvicorn
+    from uvicorn.config import LOGGING_CONFIG
+
+    log_config = LOGGING_CONFIG
+    log_config["root"] = {"level": "INFO"}
 
-    uvicorn.run(web.app, port=port, host=host)
+    uvicorn.run(app="datacontract.api:app", port=port, host=host, reload=True, log_config=LOGGING_CONFIG)
 
 
 def _handle_result(run):

datacontract/export/custom_converter.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Model,
+)
+
+
+class CustomExporter(Exporter):
+    """Exporter implementation for converting data contracts to Markdown."""
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model: Model,
+        server: str,
+        sql_server_type: str,
+        export_args: dict,
+    ) -> str:
+        """Exports a data contract to custom format with Jinja."""
+        template = export_args.get("template")
+        if template is None:
+            raise RuntimeError("Export to custom requires template argument.")
+
+        return to_custom(data_contract, template)
+
+
+def to_custom(data_contract: DataContractSpecification, template_path: Path) -> str:
+    template = get_template(template_path)
+    rendered_sql = template.render(data_contract=data_contract)
+    return rendered_sql
+
+
+def get_template(path: Path):
+    abosolute_path = Path(path).resolve()
+    env = Environment(loader=FileSystemLoader(str(abosolute_path.parent)))
+    return env.get_template(path.name)

datacontract/export/exporter.py

@@ -45,6 +45,7 @@ class ExportFormat(str, Enum):
     dcs = "dcs"
     markdown = "markdown"
     iceberg = "iceberg"
+    custom = "custom"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/export/exporter_factory.py

@@ -206,3 +206,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.custom, module_path="datacontract.export.custom_converter", class_name="CustomExporter"
+)

datacontract/lint/urls.py

@@ -27,30 +27,34 @@ def fetch_resource(url: str):
 
 def _set_api_key(headers, url):
     hostname = urlparse(url).hostname
+
+    datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+    datacontract_manager_api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
+
     if hostname == "datamesh-manager.com" or hostname.endswith(".datamesh-manager.com"):
-        datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
         if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
-            print("Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
+            print("Error: Data Mesh Manager API key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
             raise DataContractException(
                 type="lint",
                 name=f"Reading data contract from {url}",
-                reason="Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
+                reason="Error: Data Mesh Manager API key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
                 engine="datacontract",
                 result="error",
             )
         headers["x-api-key"] = datamesh_manager_api_key
     elif hostname == "datacontract-manager.com" or hostname.endswith(".datacontract-manager.com"):
-        datacontract_manager_api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
         if datacontract_manager_api_key is None or datacontract_manager_api_key == "":
-            print("Error: Data Contract Manager API Key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.")
+            print("Error: Data Contract Manager API key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.")
             raise DataContractException(
                 type="lint",
                 name=f"Reading data contract from {url}",
-                reason="Error: Data Contract Manager API Key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.",
+                reason="Error: Data Contract Manager API key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.",
                 engine="datacontract",
                 result="error",
             )
         headers["x-api-key"] = datacontract_manager_api_key
-    else:
-        # do nothing
-        pass
+
+    if datamesh_manager_api_key is not None and datamesh_manager_api_key != "":
+        headers["x-api-key"] = datamesh_manager_api_key
+    if datacontract_manager_api_key is not None and datacontract_manager_api_key != "":
+        headers["x-api-key"] = datacontract_manager_api_key