datacontract-cli 0.10.6__py3-none-any.whl → 0.10.8__py3-none-any.whl

datacontract/imports/odcs_importer.py

@@ -0,0 +1,192 @@
+import datetime
+import logging
+from typing import Any, Dict, List
+import yaml
+from datacontract.model.data_contract_specification import (
+    Availability,
+    Contact,
+    DataContractSpecification,
+    Info,
+    Model,
+    Field,
+    Retention,
+    ServiceLevel,
+    Terms,
+)
+from datacontract.model.exceptions import DataContractException
+
+DATACONTRACT_TYPES = [
+    "string",
+    "text",
+    "varchar",
+    "number",
+    "decimal",
+    "numeric",
+    "int",
+    "integer",
+    "long",
+    "bigint",
+    "float",
+    "double",
+    "boolean",
+    "timestamp",
+    "timestamp_tz",
+    "timestamp_ntz",
+    "date",
+    "array",
+    "bytes",
+    "object",
+    "record",
+    "struct",
+    "null",
+]
+
+
+def import_odcs(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
+    try:
+        with open(source, "r") as file:
+            odcs_contract = yaml.safe_load(file.read())
+
+    except Exception as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse ODCS contract",
+            reason=f"Failed to parse odcs contract from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    data_contract_specification.id = odcs_contract["uuid"]
+    data_contract_specification.info = import_info(odcs_contract)
+    data_contract_specification.terms = import_terms(odcs_contract)
+    data_contract_specification.servicelevels = import_servicelevels(odcs_contract)
+    data_contract_specification.models = import_models(odcs_contract)
+
+    return data_contract_specification
+
+
+def import_info(odcs_contract: Dict[str, Any]) -> Info:
+    info = Info(title=odcs_contract.get("quantumName"), version=odcs_contract.get("version"))
+
+    if odcs_contract.get("description").get("purpose") is not None:
+        info.description = odcs_contract.get("description").get("purpose")
+
+    if odcs_contract.get("datasetDomain") is not None:
+        info.owner = odcs_contract.get("datasetDomain")
+
+    if odcs_contract.get("productDl") is not None or odcs_contract.get("productFeedbackUrl") is not None:
+        contact = Contact()
+        if odcs_contract.get("productDl") is not None:
+            contact.name = odcs_contract.get("productDl")
+        if odcs_contract.get("productFeedbackUrl") is not None:
+            contact.url = odcs_contract.get("productFeedbackUrl")
+
+        info.contact = contact
+
+    return info
+
+
+def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None:
+    if (
+        odcs_contract.get("description").get("usage") is not None
+        or odcs_contract.get("description").get("limitations") is not None
+        or odcs_contract.get("price") is not None
+    ):
+        terms = Terms()
+        if odcs_contract.get("description").get("usage") is not None:
+            terms.usage = odcs_contract.get("description").get("usage")
+        if odcs_contract.get("description").get("limitations") is not None:
+            terms.limitations = odcs_contract.get("description").get("limitations")
+        if odcs_contract.get("price") is not None:
+            terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}"
+
+        return terms
+    else:
+        return None
+
+
+def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel:
+    # find the two properties we can map (based on the examples)
+    sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else []
+    availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None)
+    retention = next((p for p in sla_properties if p["property"] == "retention"), None)
+
+    if availability is not None or retention is not None:
+        servicelevel = ServiceLevel()
+
+        if availability is not None:
+            value = availability.get("value")
+            if isinstance(value, datetime.datetime):
+                value = value.isoformat()
+            servicelevel.availability = Availability(description=value)
+
+        if retention is not None:
+            servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}")
+
+        return servicelevel
+    else:
+        return None
+
+
+def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
+    custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties"))
+
+    odcs_tables = odcs_contract.get("dataset") if odcs_contract.get("dataset") is not None else []
+    result = {}
+
+    for table in odcs_tables:
+        description = table.get("description") if table.get("description") is not None else ""
+        model = Model(description=" ".join(description.splitlines()), type="table")
+        model.fields = import_fields(table.get("columns"), custom_type_mappings)
+        result[table.get("table")] = model
+
+    return result
+
+
+def import_fields(odcs_columns: Dict[str, Any], custom_type_mappings: Dict[str, str]) -> Dict[str, Field]:
+    logger = logging.getLogger(__name__)
+    result = {}
+
+    for column in odcs_columns:
+        mapped_type = map_type(column.get("logicalType"), custom_type_mappings)
+        if mapped_type is not None:
+            description = column.get("description") if column.get("description") is not None else ""
+            field = Field(
+                description=" ".join(description.splitlines()),
+                type=mapped_type,
+                title=column.get("businessName") if column.get("businessName") is not None else "",
+                required=not column.get("isNullable") if column.get("isNullable") is not None else False,
+                primary=column.get("isPrimary") if column.get("isPrimary") is not None else False,
+                unique=column.get("isUnique") if column.get("isUnique") is not None else False,
+                classification=column.get("classification") if column.get("classification") is not None else "",
+                tags=column.get("tags") if column.get("tags") is not None else [],
+            )
+            result[column["column"]] = field
+        else:
+            logger.info(
+                f"Can't properly map {column.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{column.get('logicalName')}' that defines your expected type as the 'value'"
+            )
+
+    return result
+
+
+def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
+    t = odcs_type.lower()
+    if t in DATACONTRACT_TYPES:
+        return t
+    elif custom_mappings.get(t) is not None:
+        return custom_mappings.get(t)
+    else:
+        return None
+
+
+def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]:
+    result = {}
+    if odcs_custom_properties is not None:
+        for prop in odcs_custom_properties:
+            if prop["property"].startswith("dc_mapping_"):
+                odcs_type_name = prop["property"].substring(11)
+                datacontract_type = prop["value"]
+                result[odcs_type_name] = datacontract_type
+
+    return result

datacontract/imports/unity_importer.py

@@ -0,0 +1,138 @@
+import json
+import requests
+import os
+import typing
+
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.exceptions import DataContractException
+
+def import_unity_from_json(
+    data_contract_specification: DataContractSpecification, source: str
+) -> DataContractSpecification:
+    try:
+        with open(source, "r") as file:
+            unity_schema = json.loads(file.read())
+    except json.JSONDecodeError as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse unity schema",
+            reason=f"Failed to parse unity schema from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+    return convert_unity_schema(data_contract_specification, unity_schema)
+
+def import_unity_from_api(
+    data_contract_specification: DataContractSpecification,
+    unity_table_full_name: typing.Optional[str] = None
+) -> DataContractSpecification:
+    databricks_instance = os.getenv('DATABRICKS_IMPORT_INSTANCE')
+    access_token = os.getenv('DATABRICKS_IMPORT_ACCESS_TOKEN')
+
+    if not databricks_instance or not access_token:
+        print("Missing environment variables for Databricks instance or access token.")
+        print("Both, $DATABRICKS_IMPORT_INSTANCE and $DATABRICKS_IMPORT_ACCESS_TOKEN must be set.")
+        exit(1)  # Exit if variables are not set
+
+    api_url = f'{databricks_instance}/api/2.1/unity-catalog/tables/{unity_table_full_name}'
+
+    headers = {
+        'Authorization': f'Bearer {access_token}'
+    }
+    response = requests.get(api_url, headers=headers)
+
+    if response.status_code != 200:
+        raise DataContractException(
+            type="schema",
+            name="Retrieve unity catalog schema",
+            reason=f"Failed to retrieve unity catalog schema from databricks instance: {response.status_code} {response.text}",
+            engine="datacontract"
+        )
+
+    convert_unity_schema(data_contract_specification, response.json())
+
+    return data_contract_specification
+
+def convert_unity_schema(
+    data_contract_specification: DataContractSpecification, unity_schema: dict
+) -> DataContractSpecification:
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    fields = import_table_fields(unity_schema.get("columns"))
+
+    table_id = unity_schema.get("table_id")
+
+    data_contract_specification.models[table_id] = Model(fields=fields, type="table")
+
+    if unity_schema.get("name") is not None:
+        data_contract_specification.models[table_id].title = unity_schema.get("name")
+
+    return data_contract_specification
+
+
+def import_table_fields(table_fields):
+    imported_fields = {}
+    for field in table_fields:
+        field_name = field.get("name")
+        imported_fields[field_name] = Field()
+        imported_fields[field_name].required = field.get("nullable") == "false"
+        imported_fields[field_name].description = field.get("comment")
+
+        # databricks api 2.1 specifies that type_name can be any of:
+        # BOOLEAN | BYTE | SHORT | INT | LONG | FLOAT | DOUBLE | DATE | TIMESTAMP | TIMESTAMP_NTZ | STRING
+        # | BINARY | DECIMAL | INTERVAL | ARRAY | STRUCT | MAP | CHAR | NULL | USER_DEFINED_TYPE | TABLE_TYPE
+        if field.get("type_name") in ["INTERVAL", "ARRAY", "STRUCT", "MAP", "USER_DEFINED_TYPE", "TABLE_TYPE"]:
+            # complex types are not supported, yet
+            raise DataContractException(
+                type="schema",
+                result="failed",
+                name="Map unity type to data contract type",
+                reason=f"type ${field.get('type_name')} is not supported yet for unity import",
+                engine="datacontract",
+            )
+
+        imported_fields[field_name].type = map_type_from_unity(field.get("type_name"))
+
+    return imported_fields
+
+
+def map_type_from_unity(type_str: str):
+    if type_str == "BOOLEAN":
+        return "boolean"
+    elif type_str == "BYTE":
+        return "bytes"
+    elif type_str == "SHORT":
+        return "int"
+    elif type_str == "INT":
+        return "int"
+    elif type_str == "LONG":
+        return "long"
+    elif type_str == "FLOAT":
+        return "float"
+    elif type_str == "DOUBLE":
+        return "double"
+    elif type_str == "DATE":
+        return "date"
+    elif type_str == "TIMESTAMP":
+        return "timestamp"
+    elif type_str == "TIMESTAMP_NTZ":
+        return "timestamp_ntz"
+    elif type_str == "STRING":
+        return "string"
+    elif type_str == "BINARY":
+        return "bytes"
+    elif type_str == "DECIMAL":
+        return "decimal"
+    elif type_str == "CHAR":
+        return "varchar"
+    elif type_str == "NULL":
+        return "null"
+    else:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="Map unity type to data contract type",
+            reason=f"Unsupported type {type_str} in unity json definition.",
+            engine="datacontract",
+        )

datacontract/model/data_contract_specification.py

@@ -39,6 +39,7 @@ class Terms(pyd.BaseModel):
     limitations: str = None
     billing: str = None
     noticePeriod: str = None
+    description: str = None
 
 
 class Definition(pyd.BaseModel):
@@ -98,6 +99,7 @@ class Model(pyd.BaseModel):
     namespace: str = None
     title: str = None
     fields: Dict[str, Field] = {}
+    config: Dict[str, Any] = None
 
 
 class Info(pyd.BaseModel):

datacontract/templates/partials/server.html

@@ -1,16 +1,20 @@
 <li class="relative flex gap-x-6 px-4 py-5 sm:px-6">
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Server</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server_name}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Server</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server_name}}</dd>
+      </div>
     </div>
   </div>
 
   {% if server.type %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Type</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.type}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Type</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.type}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -18,8 +22,10 @@
   {% if server.project %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Project</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.project}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Project</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.project}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -27,8 +33,10 @@
   {% if server.dataset %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Dataset</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.dataset}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Dataset</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.dataset}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -36,8 +44,10 @@
   {% if server.location %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Location</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.location}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Location</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.location}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -45,8 +55,10 @@
   {% if server.endpointUrl %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Endpoint URL</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.endpointUrl}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Endpoint URL</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.endpointUrl}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -54,8 +66,10 @@
   {% if server.account %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Account</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.account}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Account</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.account}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -63,8 +77,10 @@
   {% if server.host %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Host</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.host}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Host</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.host}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -72,8 +88,10 @@
   {% if server.port %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Port</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.port}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Port</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.port}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -81,8 +99,10 @@
   {% if server.catalog %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Catalog</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.catalog}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Catalog</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.catalog}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -90,8 +110,10 @@
   {% if server.database %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Database</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.database}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Database</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.database}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -99,8 +121,10 @@
   {% if server.schema_ %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Schema</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.schema_}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Schema</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.schema_}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -108,8 +132,10 @@
   {% if server.topic %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Topic</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.topic}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Topic</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.topic}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -117,8 +143,10 @@
   {% if server.path %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Path</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.path}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Path</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.path}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -126,8 +154,10 @@
   {% if server.format %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Format</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.format}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Format</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.format}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}
@@ -135,8 +165,10 @@
   {% if server.delimiter %}
   <div class="flex items-center gap-x-4">
     <div class="hidden sm:flex sm:flex-col">
-      <dt class="text-sm font-medium text-gray-500">Delimiter</dt>
-      <dd class="mt-1 text-sm text-gray-900">{{server.delimiter}}</dd>
+      <div class="flex flex-col">
+        <dt class="text-sm font-medium text-gray-500">Delimiter</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{server.delimiter}}</dd>
+      </div>
     </div>
   </div>
   {% endif %}

datacontract/templates/style/output.css

@@ -866,6 +866,11 @@ video {
        column-gap: 1rem;
 }
 
+.gap-x-6 {
+  -moz-column-gap: 1.5rem;
+       column-gap: 1.5rem;
+}
+
 .gap-y-6 {
   row-gap: 1.5rem;
 }
@@ -1399,6 +1404,10 @@ video {
     flex-direction: row;
   }
 
+  .sm\:flex-col {
+    flex-direction: column;
+  }
+
   .sm\:flex-wrap {
     flex-wrap: wrap;
   }

datacontract/web.py

@@ -1,14 +1,68 @@
-from typing import Annotated, Union
+from typing import Annotated, Union, Optional
 
+import typer
 from fastapi import FastAPI, File
+from fastapi.responses import HTMLResponse
 
-from datacontract.data_contract import DataContract
+from datacontract.data_contract import DataContract, ExportFormat
+from fastapi.responses import PlainTextResponse
 
 app = FastAPI()
 
 
+@app.get("/", response_class=HTMLResponse)
+def index():
+    # TODO OpenAPI spec
+    return """
+    <html>
+    <body>
+    <h1>datacontract web server</h1>
+    <ul>
+    <li>POST /lint</li>
+    <li>POST /export</li>
+    </ul>
+    </body>
+    </html>
+    """
+
+
 @app.post("/lint")
 def lint(file: Annotated[bytes, File()], linters: Union[str, set[str]] = "all"):
     data_contract = DataContract(data_contract_str=str(file, encoding="utf-8"))
     lint_result = data_contract.lint(enabled_linters=linters)
     return {"result": lint_result.result, "checks": lint_result.checks}
+
+
+@app.post("/export", response_class=PlainTextResponse)
+def export(
+    file: Annotated[bytes, File()],
+    export_format: Annotated[ExportFormat, typer.Option(help="The export format.")],
+    server: Annotated[str, typer.Option(help="The server name to export.")] = None,
+    model: Annotated[
+        str,
+        typer.Option(
+            help="Use the key of the model in the data contract yaml file "
+            "to refer to a model, e.g., `orders`, or `all` for all "
+            "models (default)."
+        ),
+    ] = "all",
+    rdf_base: Annotated[
+        Optional[str],
+        typer.Option(help="[rdf] The base URI used to generate the RDF graph.", rich_help_panel="RDF Options"),
+    ] = None,
+    sql_server_type: Annotated[
+        Optional[str],
+        typer.Option(
+            help="[sql] The server type to determine the sql dialect. By default, it uses 'auto' to automatically detect the sql dialect via the specified servers in the data contract.",
+            rich_help_panel="SQL Options",
+        ),
+    ] = "auto",
+):
+    result = DataContract(data_contract_str=str(file, encoding="utf-8"), server=server).export(
+        export_format=export_format,
+        model=model,
+        rdf_base=rdf_base,
+        sql_server_type=sql_server_type,
+    )
+
+    return result