datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/__init__.py

@@ -0,0 +1,13 @@
+# Configuration so that yaml.safe_dump dumps strings with line breaks with yaml literal |
+import yaml
+
+yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str
+
+
+def repr_str(dumper, data):
+    if "\n" in data:
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+    return dumper.org_represent_str(data)
+
+
+yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)

datacontract/api.py

@@ -0,0 +1,260 @@
+import logging
+import os
+from typing import Annotated, Optional
+
+import typer
+from fastapi import Body, Depends, FastAPI, HTTPException, Query, status
+from fastapi.responses import PlainTextResponse
+from fastapi.security.api_key import APIKeyHeader
+
+from datacontract.data_contract import DataContract, ExportFormat
+from datacontract.model.run import Run
+
+DATA_CONTRACT_EXAMPLE_PAYLOAD = """dataContractSpecification: 1.2.1
+id: urn:datacontract:checkout:orders-latest
+info:
+  title: Orders Latest
+  version: 2.0.0
+  owner: Sales Team
+servers:
+  production:
+    type: s3
+    location: s3://datacontract-example-orders-latest/v2/{model}/*.json
+    format: json
+    delimiter: new_line
+models:
+  orders:
+    description: One record per order. Includes cancelled and deleted orders.
+    type: table
+    fields:
+      order_id:
+        type: string
+        primaryKey: true
+      order_timestamp:
+        description: The business timestamp in UTC when the order was successfully registered in the source system and the payment was successful.
+        type: timestamp
+        required: true
+        examples:
+          - "2024-09-09T08:30:00Z"
+      order_total:
+        description: Total amount the smallest monetary unit (e.g., cents).
+        type: long
+        required: true
+        examples:
+          - 9999
+        quality:
+          - type: sql
+            description: 95% of all order total values are expected to be between 10 and 499 EUR.
+            query: |
+              SELECT quantile_cont(order_total, 0.95) AS percentile_95
+              FROM orders
+            mustBeBetween: [1000, 99900]
+      customer_id:
+        description: Unique identifier for the customer.
+        type: text
+        minLength: 10
+        maxLength: 20
+"""
+
+app = FastAPI(
+    docs_url="/",
+    title="Data Contract CLI API",
+    summary="You can use the API to test, export, and lint your data contracts.",
+    license_info={
+        "name": "MIT License",
+        "identifier": "MIT",
+    },
+    contact={"name": "Data Contract CLI", "url": "https://cli.datacontract.com/"},
+    openapi_tags=[
+        {
+            "name": "test",
+            "externalDocs": {
+                "description": "Documentation",
+                "url": "https://cli.datacontract.com/#test",
+            },
+        },
+        {
+            "name": "lint",
+            "externalDocs": {
+                "description": "Documentation",
+                "url": "https://cli.datacontract.com/#lint",
+            },
+        },
+        {
+            "name": "export",
+            "externalDocs": {
+                "description": "Documentation",
+                "url": "https://cli.datacontract.com/#export",
+            },
+        },
+    ],
+)
+
+api_key_header = APIKeyHeader(
+    name="x-api-key",
+    auto_error=False,  # this makes authentication optional
+)
+
+
+def check_api_key(api_key_header: str | None):
+    correct_api_key = os.getenv("DATACONTRACT_CLI_API_KEY")
+    if correct_api_key is None or correct_api_key == "":
+        logging.info("Environment variable DATACONTRACT_CLI_API_KEY is not set. Skip API key check.")
+        return
+    if api_key_header is None or api_key_header == "":
+        logging.info("The API key is missing.")
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing API key. Use Header 'x-api-key' to provide the API key.",
+        )
+    if api_key_header != correct_api_key:
+        logging.info("The provided API key is not correct.")
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="The provided API key is not correct.",
+        )
+    logging.info("Request authenticated with API key.")
+    pass
+
+
+@app.post(
+    "/test",
+    tags=["test"],
+    summary="Run data contract tests",
+    description="""
+              Run schema and quality tests. Data Contract CLI connects to the data sources configured in the server section.
+              This usually requires credentials to access the data sources.
+              Credentials must be provided via environment variables when running the web server.
+              POST the data contract YAML as payload.
+            """,
+    responses={
+        401: {
+            "description": "Unauthorized (when an environment variable DATACONTRACT_CLI_API_KEY is configured).",
+            "content": {
+                "application/json": {
+                    "examples": {
+                        "api_key_missing": {
+                            "summary": "API key Missing",
+                            "value": {"detail": "Missing API key. Use Header 'x-api-key' to provide the API key."},
+                        },
+                        "api_key_wrong": {
+                            "summary": "API key Wrong",
+                            "value": {"detail": "The provided API key is not correct."},
+                        },
+                    }
+                }
+            },
+        },
+    },
+    response_model_exclude_none=True,
+    response_model_exclude_unset=True,
+)
+async def test(
+    body: Annotated[
+        str,
+        Body(
+            title="Data Contract YAML",
+            media_type="application/yaml",
+            examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
+        ),
+    ],
+    api_key: Annotated[str | None, Depends(api_key_header)] = None,
+    server: Annotated[
+        str | None,
+        Query(
+            description="The server name to test. Optional, if there is only one server.",
+            examples=["production"],
+        ),
+    ] = None,
+    publish_url: Annotated[
+        str | None,
+        Query(
+            description="URL to publish test results. Optional, if you want to publish the test results to a Data Mesh Manager or Data Contract Manager. Example: https://api.datamesh-manager.com/api/test-results",
+            examples=["https://api.datamesh-manager.com/api/test-results"],
+        ),
+    ] = None,
+) -> Run:
+    check_api_key(api_key)
+    logging.info("Testing data contract...")
+    logging.info(body)
+    return DataContract(data_contract_str=body, server=server, publish_url=publish_url).test()
+
+
+@app.post(
+    "/lint",
+    tags=["lint"],
+    summary="Validate that the datacontract.yaml is correctly formatted.",
+    description="""Validate that the datacontract.yaml is correctly formatted.""",
+)
+async def lint(
+    body: Annotated[
+        str,
+        Body(
+            title="Data Contract YAML",
+            media_type="application/yaml",
+            examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
+        ),
+    ],
+    schema: Annotated[
+        str | None,
+        Query(
+            examples=["https://datacontract.com/datacontract.schema.json"],
+            description="The schema to use for validation. This must be a URL.",
+        ),
+    ] = None,
+):
+    data_contract = DataContract(data_contract_str=body, schema_location=schema)
+    lint_result = data_contract.lint()
+    return {"result": lint_result.result, "checks": lint_result.checks}
+
+
+@app.post(
+    "/export",
+    tags=["export"],
+    summary="Convert data contract to a specific format.",
+    response_class=PlainTextResponse,
+)
+def export(
+    body: Annotated[
+        str,
+        Body(
+            title="Data Contract YAML",
+            media_type="application/yaml",
+            examples=[DATA_CONTRACT_EXAMPLE_PAYLOAD],
+        ),
+    ],
+    format: Annotated[ExportFormat, typer.Option(help="The export format.")],
+    server: Annotated[
+        str | None,
+        Query(
+            examples=["production"],
+            description="The server name to export. Optional, if there is only one server.",
+        ),
+    ] = None,
+    model: Annotated[
+        str | None,
+        Query(
+            description="Use the key of the model in the data contract yaml file "
+            "to refer to a model, e.g., `orders`, or `all` for all "
+            "models (default).",
+        ),
+    ] = "all",
+    rdf_base: Annotated[
+        Optional[str],
+        typer.Option(help="[rdf] The base URI used to generate the RDF graph.", rich_help_panel="RDF Options"),
+    ] = None,
+    sql_server_type: Annotated[
+        Optional[str],
+        Query(
+            description="[sql] The server type to determine the sql dialect. By default, it uses 'auto' to automatically detect the sql dialect via the specified servers in the data contract.",
+        ),
+    ] = None,
+):
+    result = DataContract(data_contract_str=body, server=server).export(
+        export_format=format,
+        model=model,
+        rdf_base=rdf_base,
+        sql_server_type=sql_server_type,
+    )
+
+    return result

datacontract/breaking/breaking.py

@@ -1,11 +1,223 @@
+from datacontract.breaking.breaking_change import BreakingChange, Location, Severity
 from datacontract.breaking.breaking_rules import BreakingRules
-from datacontract.model.breaking_change import BreakingChange, Location, Severity
-from datacontract.model.data_contract_specification import Field, Model, Quality
+from datacontract.model.data_contract_specification import Contact, DeprecatedQuality, Field, Info, Model, Terms
+
+
+def info_breaking_changes(
+    old_info: Info,
+    new_info: Info,
+    new_path: str,
+    include_severities: [Severity],
+) -> list[BreakingChange]:
+    results = list[BreakingChange]()
+
+    composition = ["info"]
+
+    if old_info and new_info:
+        info_definition_fields = vars(new_info) | new_info.model_extra | old_info.model_extra
+
+        for info_definition_field in info_definition_fields.keys():
+            if info_definition_field == "contact":
+                continue
+
+            old_value = getattr(old_info, info_definition_field, None)
+            new_value = getattr(new_info, info_definition_field, None)
+
+            rule_name = None
+            description = None
+
+            if old_value is None and new_value is not None:
+                rule_name = f"info_{_camel_to_snake(info_definition_field)}_added"
+                description = f"added with value: `{new_value}`"
+
+            elif old_value is not None and new_value is None:
+                rule_name = f"info_{_camel_to_snake(info_definition_field)}_removed"
+                description = "removed info property"
+
+            elif old_value != new_value:
+                rule_name = f"info_{_camel_to_snake(info_definition_field)}_updated"
+                description = f"changed from `{old_value}` to `{new_value}`"
+
+            if rule_name is not None:
+                severity = _get_rule(rule_name)
+                if severity in include_severities:
+                    results.append(
+                        BreakingChange(
+                            description=description,
+                            check_name=rule_name,
+                            severity=severity,
+                            location=Location(path=new_path, composition=composition + [info_definition_field]),
+                        )
+                    )
+
+        results.extend(
+            contact_breaking_changes(
+                old_contact=getattr(old_info, "contact", None),
+                new_contact=getattr(new_info, "contact", None),
+                composition=composition + ["contact"],
+                new_path=new_path,
+                include_severities=include_severities,
+            )
+        )
+
+    return results
+
+
+def contact_breaking_changes(
+    old_contact: Contact,
+    new_contact: Contact,
+    composition: list[str],
+    new_path: str,
+    include_severities: [Severity],
+) -> list[BreakingChange]:
+    results = list[BreakingChange]()
+
+    if not old_contact and new_contact:
+        rule_name = "contact_added"
+        severity = _get_rule(rule_name)
+        description = "added contact"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+
+    elif old_contact and not new_contact:
+        rule_name = "contact_removed"
+        severity = _get_rule(rule_name)
+        description = "removed contact"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+
+    elif old_contact and new_contact:
+        contact_definition_fields = vars(new_contact) | new_contact.model_extra | old_contact.model_extra
+
+        for contact_definition_field in contact_definition_fields.keys():
+            old_value = getattr(old_contact, contact_definition_field, None)
+            new_value = getattr(new_contact, contact_definition_field, None)
+
+            rule_name = None
+            description = None
+
+            if old_value is None and new_value is not None:
+                rule_name = f"contact_{_camel_to_snake(contact_definition_field)}_added"
+                description = f"added with value: `{new_value}`"
+
+            elif old_value is not None and new_value is None:
+                rule_name = f"contact_{_camel_to_snake(contact_definition_field)}_removed"
+                description = "removed contact property"
+
+            elif old_value != new_value:
+                rule_name = f"contact_{_camel_to_snake(contact_definition_field)}_updated"
+                description = f"changed from `{old_value}` to `{new_value}`"
+
+            if rule_name is not None:
+                severity = _get_rule(rule_name)
+                if severity in include_severities:
+                    results.append(
+                        BreakingChange(
+                            description=description,
+                            check_name=rule_name,
+                            severity=severity,
+                            location=Location(path=new_path, composition=composition + [contact_definition_field]),
+                        )
+                    )
+
+    return results
+
+
+def terms_breaking_changes(
+    old_terms: Terms,
+    new_terms: Terms,
+    new_path: str,
+    include_severities: [Severity],
+) -> list[BreakingChange]:
+    results = list[BreakingChange]()
+
+    composition = ["terms"]
+
+    if not old_terms and new_terms:
+        rule_name = "terms_added"
+        severity = _get_rule(rule_name)
+        description = "added terms"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+    elif old_terms and not new_terms:
+        rule_name = "terms_removed"
+        severity = _get_rule(rule_name)
+        description = "removed terms"
+
+        if severity in include_severities:
+            results.append(
+                BreakingChange(
+                    description=description,
+                    check_name=rule_name,
+                    severity=severity,
+                    location=Location(path=new_path, composition=composition),
+                )
+            )
+
+    if old_terms and new_terms:
+        terms_definition_fields = vars(new_terms) | new_terms.model_extra | old_terms.model_extra
+
+        for terms_definition_field in terms_definition_fields.keys():
+            old_value = getattr(old_terms, terms_definition_field, None)
+            new_value = getattr(new_terms, terms_definition_field, None)
+
+            rule_name = None
+            description = None
+
+            if old_value is None and new_value is not None:
+                rule_name = f"terms_{_camel_to_snake(terms_definition_field)}_added"
+                description = f"added with value: `{new_value}`"
+
+            elif old_value is not None and new_value is None:
+                rule_name = f"terms_{_camel_to_snake(terms_definition_field)}_removed"
+                description = "removed info property"
+
+            elif old_value != new_value:
+                rule_name = f"terms_{_camel_to_snake(terms_definition_field)}_updated"
+                description = f"changed from `{old_value}` to `{new_value}`"
+
+            if rule_name is not None:
+                severity = _get_rule(rule_name)
+                if severity in include_severities:
+                    results.append(
+                        BreakingChange(
+                            description=description,
+                            check_name=rule_name,
+                            severity=severity,
+                            location=Location(path=new_path, composition=composition + [terms_definition_field]),
+                        )
+                    )
+
+    return results
 
 
 def quality_breaking_changes(
-    old_quality: Quality,
-    new_quality: Quality,
+    old_quality: DeprecatedQuality,
+    new_quality: DeprecatedQuality,
     new_path: str,
     include_severities: [Severity],
 ) -> list[BreakingChange]:
@@ -129,14 +341,14 @@ def model_breaking_changes(
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
-    model_definition_fields = vars(new_model)
+    model_definition_fields = vars(new_model) | new_model.model_extra | old_model.model_extra
 
     for model_definition_field in model_definition_fields.keys():
         if model_definition_field == "fields":
             continue
 
-        old_value = getattr(old_model, model_definition_field)
-        new_value = getattr(new_model, model_definition_field)
+        old_value = getattr(old_model, model_definition_field, None)
+        new_value = getattr(new_model, model_definition_field, None)
 
         rule_name = None
         description = None
@@ -237,13 +449,13 @@ def field_breaking_changes(
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
-    field_definition_fields = vars(new_field)
+    field_definition_fields = vars(new_field) | new_field.model_extra | old_field.model_extra
     for field_definition_field in field_definition_fields.keys():
         if field_definition_field == "ref_obj":
             continue
 
-        old_value = getattr(old_field, field_definition_field)
-        new_value = getattr(new_field, field_definition_field)
+        old_value = getattr(old_field, field_definition_field, None)
+        new_value = getattr(new_field, field_definition_field, None)
 
         if field_definition_field == "fields":
             results.extend(
@@ -257,6 +469,18 @@ def field_breaking_changes(
             )
             continue
 
+        if field_definition_field == "items" and old_field.type == "array" and new_field.type == "array":
+            results.extend(
+                field_breaking_changes(
+                    old_field=old_value,
+                    new_field=new_value,
+                    composition=composition + ["items"],
+                    new_path=new_path,
+                    include_severities=include_severities,
+                )
+            )
+            continue
+
         rule_name = None
         description = None
 
@@ -308,9 +532,15 @@ def _get_rule(rule_name) -> Severity:
     try:
         return getattr(BreakingRules, rule_name)
     except AttributeError:
-        print(f"WARNING: Breaking Rule not found for {rule_name}!")
-        return Severity.ERROR
+        try:
+            first, *_, last = rule_name.split("_")
+            short_rule = "__".join([first, last])
+            return getattr(BreakingRules, short_rule)
+        except AttributeError:
+            print(f"WARNING: Breaking Rule not found for {rule_name}!")
+            return Severity.ERROR
 
 
 def _camel_to_snake(s):
+    s = s.replace("-", "_")
     return "".join(["_" + c.lower() if c.isupper() else c for c in s]).lstrip("_")

datacontract/breaking/breaking_rules.py

@@ -1,4 +1,4 @@
-from datacontract.model.breaking_change import Severity
+from datacontract.breaking.breaking_change import Severity
 
 
 class BreakingRules:
@@ -12,6 +12,10 @@ class BreakingRules:
 
     model_type_updated = Severity.ERROR
 
+    model__removed = Severity.INFO  # To support model extension keys
+    model__added = Severity.INFO
+    model__updated = Severity.INFO
+
     # field rules
     field_added = Severity.INFO
     field_removed = Severity.ERROR
@@ -20,6 +24,10 @@ class BreakingRules:
     field_ref_removed = Severity.WARNING
     field_ref_updated = Severity.WARNING
 
+    field_title_added = Severity.INFO
+    field_title_removed = Severity.INFO
+    field_title_updated = Severity.INFO
+
     field_type_added = Severity.WARNING
     field_type_removed = Severity.WARNING
     field_type_updated = Severity.ERROR
@@ -30,8 +38,14 @@ class BreakingRules:
 
     field_required_updated = Severity.ERROR
 
+    field_primary_added = Severity.WARNING
+    field_primary_removed = Severity.WARNING
     field_primary_updated = Severity.WARNING
 
+    field_primary_key_added = Severity.WARNING
+    field_primary_key_removed = Severity.WARNING
+    field_primary_key_updated = Severity.WARNING
+
     field_references_added = Severity.WARNING
     field_references_removed = Severity.WARNING
     field_references_updated = Severity.WARNING
@@ -86,9 +100,31 @@ class BreakingRules:
     field_tags_removed = Severity.INFO
     field_tags_updated = Severity.INFO
 
+    field_example_added = Severity.INFO
+    field_example_updated = Severity.INFO
+    field_example_removed = Severity.INFO
+
+    field__removed = Severity.INFO  # To support field extension keys
+    field__added = Severity.INFO
+    field__updated = Severity.INFO
+
     # quality Rules
     quality_added = Severity.INFO
     quality_removed = Severity.WARNING
 
     quality_type_updated = Severity.WARNING
     quality_specification_updated = Severity.WARNING
+
+    # info rules
+    info__added = Severity.INFO  # will match `info_<somekey>_added` etc
+    info__removed = Severity.INFO
+    info__updated = Severity.INFO
+
+    contact__added = Severity.INFO
+    contact__removed = Severity.INFO
+    contact__updated = Severity.INFO
+
+    # terms rules
+    terms__added = Severity.INFO
+    terms__removed = Severity.INFO
+    terms__updated = Severity.INFO