datacontract-cli 0.10.20__py3-none-any.whl → 0.10.21__py3-none-any.whl

datacontract/{web.py → api.py}

@@ -1,8 +1,11 @@
+import logging
+import os
 from typing import Annotated, Optional
 
 import typer
-from fastapi import Body, FastAPI, Query
+from fastapi import Body, Depends, FastAPI, HTTPException, Query, status
 from fastapi.responses import PlainTextResponse
+from fastapi.security.api_key import APIKeyHeader
 
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.model.run import Run
@@ -55,8 +58,8 @@ models:
 
 app = FastAPI(
     docs_url="/",
-    title="Data Contract API",
-    summary="API to execute Data Contract CLI operations.",
+    title="Data Contract CLI API",
+    summary="You can use the API to test, export, and lint your data contracts.",
     license_info={
         "name": "MIT License",
         "identifier": "MIT",
@@ -87,6 +90,32 @@ app = FastAPI(
     ],
 )
 
+api_key_header = APIKeyHeader(
+    name="x-api-key",
+    auto_error=False,  # this makes authentication optional
+)
+
+
+def check_api_key(api_key_header: str | None):
+    correct_api_key = os.getenv("DATACONTRACT_CLI_API_KEY")
+    if correct_api_key is None or correct_api_key == "":
+        logging.info("Environment variable DATACONTRACT_CLI_API_KEY is not set. Skip API key check.")
+        return
+    if api_key_header is None or api_key_header == "":
+        logging.info("The API key is missing.")
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing API key. Use Header 'x-api-key' to provide the API key.",
+        )
+    if api_key_header != correct_api_key:
+        logging.info("The provided API key is not correct.")
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="The provided API key is not correct.",
+        )
+    logging.info("Request authenticated with API key.")
+    pass
+
 
 @app.post(
     "/test",
@@ -98,6 +127,25 @@ app = FastAPI(
               Credentials must be provided via environment variables when running the web server.
               POST the data contract YAML as payload.
             """,
+    responses={
+        401: {
+            "description": "Unauthorized (when an environment variable DATACONTRACT_CLI_API_KEY is configured).",
+            "content": {
+                "application/json": {
+                    "examples": {
+                        "api_key_missing": {
+                            "summary": "API key Missing",
+                            "value": {"detail": "Missing API key. Use Header 'x-api-key' to provide the API key."},
+                        },
+                        "api_key_wrong": {
+                            "summary": "API key Wrong",
+                            "value": {"detail": "The provided API key is not correct."},
+                        },
+                    }
+                }
+            },
+        },
+    },
     response_model_exclude_none=True,
     response_model_exclude_unset=True,
 )
@@ -110,6 +158,7 @@ async def test(
             examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
         ),
     ],
+    api_key: Annotated[str | None, Depends(api_key_header)] = None,
     server: Annotated[
         str | None,
         Query(
@@ -118,6 +167,9 @@ async def test(
         ),
     ] = None,
 ) -> Run:
+    check_api_key(api_key)
+    logging.info("Testing data contract...")
+    logging.info(body)
     return DataContract(data_contract_str=body, server=server).test()
 
 

datacontract/cli.py

@@ -196,6 +196,11 @@ def export(
         Optional[str],
         typer.Option(help="[engine] The engine used for great expection run."),
     ] = None,
+    # TODO: this should be a subcommand
+    template: Annotated[
+        Optional[Path],
+        typer.Option(help="[custom] The file path of Jinja template."),
+    ] = None,
 ):
     """
     Convert data contract to a specific format. Saves to file specified by `output` option if present, otherwise prints to stdout.
@@ -208,6 +213,7 @@ def export(
         rdf_base=rdf_base,
         sql_server_type=sql_server_type,
         engine=engine,
+        template=template,
     )
     # Don't interpret console markup in output.
     if output is None:
@@ -344,7 +350,7 @@ def catalog(
     ] = None,
 ):
     """
-    Create an html catalog of data contracts.
+    Create a html catalog of data contracts.
     """
     path = Path(output)
     path.mkdir(parents=True, exist_ok=True)
@@ -433,16 +439,32 @@ def diff(
 
 
 @app.command()
-def serve(
+def api(
     port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4242,
-    host: Annotated[str, typer.Option(help="Bind socket to this host.")] = "127.0.0.1",
+    host: Annotated[
+        str, typer.Option(help="Bind socket to this host. Hint: For running in docker, set it to 0.0.0.0")
+    ] = "127.0.0.1",
 ):
     """
-    Start the datacontract web server.
+    Start the datacontract CLI as server application with REST API.
+
+    The OpenAPI documentation as Swagger UI is available on http://localhost:4242.
+    You can execute the commands directly from the Swagger UI.
+
+    To protect the API, you can set the environment variable DATACONTRACT_CLI_API_KEY to a secret API key.
+    To authenticate, requests must include the header 'x-api-key' with the correct API key.
+    This is highly recommended, as data contract tests may be subject to SQL injections or leak sensitive information.
+
+    To connect to servers (such as a Snowflake data source), set the credentials as environment variables as documented in
+    https://cli.datacontract.com/#test
     """
     import uvicorn
+    from uvicorn.config import LOGGING_CONFIG
+
+    log_config = LOGGING_CONFIG
+    log_config["root"] = {"level": "INFO"}
 
-    uvicorn.run("datacontract.web:app", port=port, host=host, reload=True)
+    uvicorn.run(app="datacontract.api:app", port=port, host=host, reload=True, log_config=LOGGING_CONFIG)
 
 
 def _handle_result(run):

datacontract/export/custom_converter.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+from datacontract.export.exporter import Exporter
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Model,
+)
+
+
+class CustomExporter(Exporter):
+    """Exporter implementation for converting data contracts to Markdown."""
+
+    def export(
+        self,
+        data_contract: DataContractSpecification,
+        model: Model,
+        server: str,
+        sql_server_type: str,
+        export_args: dict,
+    ) -> str:
+        """Exports a data contract to custom format with Jinja."""
+        template = export_args.get("template")
+        if template is None:
+            raise RuntimeError("Export to custom requires template argument.")
+
+        return to_custom(data_contract, template)
+
+
+def to_custom(data_contract: DataContractSpecification, template_path: Path) -> str:
+    template = get_template(template_path)
+    rendered_sql = template.render(data_contract=data_contract)
+    return rendered_sql
+
+
+def get_template(path: Path):
+    abosolute_path = Path(path).resolve()
+    env = Environment(loader=FileSystemLoader(str(abosolute_path.parent)))
+    return env.get_template(path.name)

datacontract/export/exporter.py

@@ -45,6 +45,7 @@ class ExportFormat(str, Enum):
     dcs = "dcs"
     markdown = "markdown"
     iceberg = "iceberg"
+    custom = "custom"
 
     @classmethod
     def get_supported_formats(cls):

datacontract/export/exporter_factory.py

@@ -206,3 +206,7 @@ exporter_factory.register_lazy_exporter(
 exporter_factory.register_lazy_exporter(
     name=ExportFormat.iceberg, module_path="datacontract.export.iceberg_converter", class_name="IcebergExporter"
 )
+
+exporter_factory.register_lazy_exporter(
+    name=ExportFormat.custom, module_path="datacontract.export.custom_converter", class_name="CustomExporter"
+)

datacontract/lint/urls.py

@@ -33,22 +33,22 @@ def _set_api_key(headers, url):
 
     if hostname == "datamesh-manager.com" or hostname.endswith(".datamesh-manager.com"):
         if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
-            print("Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
+            print("Error: Data Mesh Manager API key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
             raise DataContractException(
                 type="lint",
                 name=f"Reading data contract from {url}",
-                reason="Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
+                reason="Error: Data Mesh Manager API key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
                 engine="datacontract",
                 result="error",
             )
         headers["x-api-key"] = datamesh_manager_api_key
     elif hostname == "datacontract-manager.com" or hostname.endswith(".datacontract-manager.com"):
         if datacontract_manager_api_key is None or datacontract_manager_api_key == "":
-            print("Error: Data Contract Manager API Key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.")
+            print("Error: Data Contract Manager API key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.")
             raise DataContractException(
                 type="lint",
                 name=f"Reading data contract from {url}",
-                reason="Error: Data Contract Manager API Key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.",
+                reason="Error: Data Contract Manager API key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.",
                 engine="datacontract",
                 result="error",
             )

datacontract/model/data_contract_specification.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 import pydantic as pyd
 import yaml
@@ -32,9 +32,9 @@ DATACONTRACT_TYPES = [
 
 
 class Contact(pyd.BaseModel):
-    name: str = None
-    url: str = None
-    email: str = None
+    name: str | None = None
+    url: str | None = None
+    email: str | None = None
 
     model_config = pyd.ConfigDict(
         extra="allow",
@@ -42,37 +42,37 @@ class Contact(pyd.BaseModel):
 
 
 class ServerRole(pyd.BaseModel):
-    name: str = None
-    description: str = None
+    name: str | None = None
+    description: str | None = None
     model_config = pyd.ConfigDict(
         extra="allow",
     )
 
 
 class Server(pyd.BaseModel):
-    type: str = None
-    description: str = None
-    environment: str = None
-    format: str = None
-    project: str = None
-    dataset: str = None
-    path: str = None
-    delimiter: str = None
-    endpointUrl: str = None
-    location: str = None
-    account: str = None
-    database: str = None
-    schema_: str = pyd.Field(default=None, alias="schema")
-    host: str = None
-    port: int = None
-    catalog: str = None
-    topic: str = None
-    http_path: str = None  # Use ENV variable
-    token: str = None  # Use ENV variable
-    dataProductId: str = None
-    outputPortId: str = None
-    driver: str = None
-    storageAccount: str = None
+    type: str | None = None
+    description: str | None = None
+    environment: str | None = None
+    format: str | None = None
+    project: str | None = None
+    dataset: str | None = None
+    path: str | None = None
+    delimiter: str | None = None
+    endpointUrl: str | None = None
+    location: str | None = None
+    account: str | None = None
+    database: str | None = None
+    schema_: str | None = pyd.Field(default=None, alias="schema")
+    host: str | None = None
+    port: int | None = None
+    catalog: str | None = None
+    topic: str | None = None
+    http_path: str | None = None  # Use ENV variable
+    token: str | None = None  # Use ENV variable
+    dataProductId: str | None = None
+    outputPortId: str | None = None
+    driver: str | None = None
+    storageAccount: str | None = None
     roles: List[ServerRole] = None
 
     model_config = pyd.ConfigDict(
@@ -81,11 +81,11 @@ class Server(pyd.BaseModel):
 
 
 class Terms(pyd.BaseModel):
-    usage: str = None
-    limitations: str = None
-    billing: str = None
-    noticePeriod: str = None
-    description: str = None
+    usage: str | None = None
+    limitations: str | None = None
+    billing: str | None = None
+    noticePeriod: str | None = None
+    description: str | None = None
 
     model_config = pyd.ConfigDict(
         extra="allow",
@@ -93,26 +93,27 @@ class Terms(pyd.BaseModel):
 
 
 class Definition(pyd.BaseModel):
-    domain: str = None
-    name: str = None
-    title: str = None
-    description: str = None
-    type: str = None
+    domain: str | None = None
+    name: str | None = None
+    title: str | None = None
+    description: str | None = None
+    type: str | None = None
     enum: List[str] = []
-    format: str = None
-    minLength: int = None
-    maxLength: int = None
-    pattern: str = None
-    minimum: int = None
-    exclusiveMinimum: int = None
-    maximum: int = None
-    exclusiveMaximum: int = None
-    pii: bool = None
-    classification: str = None
+    format: str | None = None
+    minLength: int | None = None
+    maxLength: int | None = None
+    pattern: str | None = None
+    minimum: int | None = None
+    exclusiveMinimum: int | None = None
+    maximum: int | None = None
+    exclusiveMaximum: int | None = None
+    pii: bool | None = None
+    classification: str | None = None
     fields: Dict[str, "Field"] = {}
+    items: "Field" = None
     tags: List[str] = []
     links: Dict[str, str] = {}
-    example: str = None
+    example: str | None = None
     examples: List[Any] | None = None
 
     model_config = pyd.ConfigDict(
@@ -121,20 +122,20 @@ class Definition(pyd.BaseModel):
 
 
 class Quality(pyd.BaseModel):
-    type: str = None
-    description: str = None
-    query: str = None
-    dialect: str = None
-    mustBe: int = None
-    mustNotBe: int = None
-    mustBeGreaterThan: int = None
-    mustBeGreaterThanOrEqualTo: int = None
-    mustBeLessThan: int = None
-    mustBeLessThanOrEqualTo: int = None
+    type: str | None = None
+    description: str | None = None
+    query: str | None = None
+    dialect: str | None = None
+    mustBe: int | None = None
+    mustNotBe: int | None = None
+    mustBeGreaterThan: int | None = None
+    mustBeGreaterThanOrEqualTo: int | None = None
+    mustBeLessThan: int | None = None
+    mustBeLessThanOrEqualTo: int | None = None
     mustBeBetween: List[int] = None
     mustNotBeBetween: List[int] = None
-    engine: str = None
-    implementation: str | Dict[str, Any] = None
+    engine: str | None = None
+    implementation: str | Dict[str, Any] | None = None
 
     model_config = pyd.ConfigDict(
         extra="allow",
@@ -144,26 +145,26 @@ class Quality(pyd.BaseModel):
 class Field(pyd.BaseModel):
     ref: str = pyd.Field(default=None, alias="$ref")
     title: str | None = None
-    type: str = None
-    format: str = None
-    required: bool = None
+    type: str | None = None
+    format: str | None = None
+    required: bool | None = None
     primary: bool = pyd.Field(
         default=None,
         deprecated="Removed in Data Contract Specification v1.1.0. Use primaryKey instead.",
     )
     primaryKey: bool | None = None
     unique: bool | None = None
-    references: str = None
+    references: str | None = None
     description: str | None = None
     pii: bool | None = None
     classification: str | None = None
-    pattern: str = None
-    minLength: int = None
-    maxLength: int = None
-    minimum: int = None
-    exclusiveMinimum: int = None
-    maximum: int = None
-    exclusiveMaximum: int = None
+    pattern: str | None = None
+    minLength: int | None = None
+    maxLength: int | None = None
+    minimum: int | None = None
+    exclusiveMinimum: int | None = None
+    maximum: int | None = None
+    exclusiveMaximum: int | None = None
     enum: List[str] | None = []
     tags: List[str] | None = []
     links: Dict[str, str] = {}
@@ -171,11 +172,11 @@ class Field(pyd.BaseModel):
     items: "Field" = None
     keys: "Field" = None
     values: "Field" = None
-    precision: int = None
-    scale: int = None
-    example: str = pyd.Field(
+    precision: int | None = None
+    scale: int | None = None
+    example: Any | None = pyd.Field(
         default=None,
-        deprecated="Removed in Data Contract Specification v1.1.0. Use " "examples instead.",
+        deprecated="Removed in Data Contract Specification v1.1.0. Use examples instead.",
     )
     examples: List[Any] | None = None
     quality: List[Quality] | None = []
@@ -187,10 +188,10 @@ class Field(pyd.BaseModel):
 
 
 class Model(pyd.BaseModel):
-    description: Optional[str] = None
-    type: Optional[str] = None
-    namespace: Optional[str] = None
-    title: Optional[str] = None
+    description: str | None = None
+    type: str | None = None
+    namespace: str | None = None
+    title: str | None = None
     fields: Dict[str, Field] = {}
     quality: List[Quality] | None = []
     primaryKey: List[str] | None = []
@@ -204,12 +205,12 @@ class Model(pyd.BaseModel):
 
 
 class Info(pyd.BaseModel):
-    title: str = None
-    version: str = None
-    status: str = None
-    description: str = None
-    owner: str = None
-    contact: Contact = None
+    title: str | None = None
+    version: str | None = None
+    status: str | None = None
+    description: str | None = None
+    owner: str | None = None
+    contact: Contact | None = None
 
     model_config = pyd.ConfigDict(
         extra="allow",
@@ -217,91 +218,91 @@ class Info(pyd.BaseModel):
 
 
 class Example(pyd.BaseModel):
-    type: str = None
-    description: str = None
-    model: str = None
+    type: str | None = None
+    description: str | None = None
+    model: str | None = None
     data: str | object = None
 
 
 # Deprecated Quality class
 class DeprecatedQuality(pyd.BaseModel):
-    type: str = None
+    type: str | None = None
     specification: str | object = None
 
 
 class Availability(pyd.BaseModel):
-    description: Optional[str] = None
-    percentage: Optional[str] = None
+    description: str | None = None
+    percentage: str | None = None
 
 
 class Retention(pyd.BaseModel):
-    description: Optional[str] = None
-    period: Optional[str] = None
-    unlimited: Optional[bool] = None
-    timestampField: Optional[str] = None
+    description: str | None = None
+    period: str | None = None
+    unlimited: bool | None = None
+    timestampField: str | None = None
 
 
 class Latency(pyd.BaseModel):
-    description: Optional[str] = None
-    threshold: Optional[str] = None
-    sourceTimestampField: Optional[str] = None
-    processedTimestampField: Optional[str] = None
+    description: str | None = None
+    threshold: str | None = None
+    sourceTimestampField: str | None = None
+    processedTimestampField: str | None = None
 
 
 class Freshness(pyd.BaseModel):
-    description: Optional[str] = None
-    threshold: Optional[str] = None
-    timestampField: Optional[str] = None
+    description: str | None = None
+    threshold: str | None = None
+    timestampField: str | None = None
 
 
 class Frequency(pyd.BaseModel):
-    description: Optional[str] = None
-    type: Optional[str] = None
-    interval: Optional[str] = None
-    cron: Optional[str] = None
+    description: str | None = None
+    type: str | None = None
+    interval: str | None = None
+    cron: str | None = None
 
 
 class Support(pyd.BaseModel):
-    description: Optional[str] = None
-    time: Optional[str] = None
-    responseTime: Optional[str] = None
+    description: str | None = None
+    time: str | None = None
+    responseTime: str | None = None
 
 
 class Backup(pyd.BaseModel):
-    description: Optional[str] = None
-    interval: Optional[str] = None
-    cron: Optional[str] = None
-    recoveryTime: Optional[str] = None
-    recoveryPoint: Optional[str] = None
+    description: str | None = None
+    interval: str | None = None
+    cron: str | None = None
+    recoveryTime: str | None = None
+    recoveryPoint: str | None = None
 
 
 class ServiceLevel(pyd.BaseModel):
-    availability: Optional[Availability] = None
-    retention: Optional[Retention] = None
-    latency: Optional[Latency] = None
-    freshness: Optional[Freshness] = None
-    frequency: Optional[Frequency] = None
-    support: Optional[Support] = None
-    backup: Optional[Backup] = None
+    availability: Availability | None = None
+    retention: Retention | None = None
+    latency: Latency | None = None
+    freshness: Freshness | None = None
+    frequency: Frequency | None = None
+    support: Support | None = None
+    backup: Backup | None = None
 
 
 class DataContractSpecification(pyd.BaseModel):
-    dataContractSpecification: str = None
-    id: str = None
-    info: Info = None
+    dataContractSpecification: str | None = None
+    id: str | None = None
+    info: Info | None = None
     servers: Dict[str, Server] = {}
-    terms: Terms = None
+    terms: Terms | None = None
     models: Dict[str, Model] = {}
     definitions: Dict[str, Definition] = {}
     examples: List[Example] = pyd.Field(
         default_factory=list,
         deprecated="Removed in Data Contract Specification " "v1.1.0. Use models.examples instead.",
     )
-    quality: DeprecatedQuality = pyd.Field(
+    quality: DeprecatedQuality | None = pyd.Field(
         default=None,
         deprecated="Removed in Data Contract Specification v1.1.0. Use " "model-level and field-level quality instead.",
     )
-    servicelevels: Optional[ServiceLevel] = None
+    servicelevels: ServiceLevel | None = None
     links: Dict[str, str] = {}
     tags: List[str] = []