datacontract-cli 0.10.7__py3-none-any.whl → 0.10.9__py3-none-any.whl

datacontract/imports/unity_importer.py

@@ -0,0 +1,152 @@
+import json
+import requests
+import os
+import typing
+
+from datacontract.imports.importer import Importer
+from datacontract.model.data_contract_specification import DataContractSpecification, Model, Field
+from datacontract.model.exceptions import DataContractException
+
+
+class UnityImporter(Importer):
+    def import_source(
+        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
+    ) -> dict:
+        if source is not None:
+            data_contract_specification = import_unity_from_json(data_contract_specification, source)
+        else:
+            data_contract_specification = import_unity_from_api(
+                data_contract_specification, import_args.get("unity_table_full_name")
+            )
+        return data_contract_specification
+
+
+def import_unity_from_json(
+    data_contract_specification: DataContractSpecification, source: str
+) -> DataContractSpecification:
+    try:
+        with open(source, "r") as file:
+            unity_schema = json.loads(file.read())
+    except json.JSONDecodeError as e:
+        raise DataContractException(
+            type="schema",
+            name="Parse unity schema",
+            reason=f"Failed to parse unity schema from {source}",
+            engine="datacontract",
+            original_exception=e,
+        )
+    return convert_unity_schema(data_contract_specification, unity_schema)
+
+
+def import_unity_from_api(
+    data_contract_specification: DataContractSpecification, unity_table_full_name: typing.Optional[str] = None
+) -> DataContractSpecification:
+    databricks_instance = os.getenv("DATABRICKS_IMPORT_INSTANCE")
+    access_token = os.getenv("DATABRICKS_IMPORT_ACCESS_TOKEN")
+
+    if not databricks_instance or not access_token:
+        print("Missing environment variables for Databricks instance or access token.")
+        print("Both, $DATABRICKS_IMPORT_INSTANCE and $DATABRICKS_IMPORT_ACCESS_TOKEN must be set.")
+        exit(1)  # Exit if variables are not set
+
+    api_url = f"{databricks_instance}/api/2.1/unity-catalog/tables/{unity_table_full_name}"
+
+    headers = {"Authorization": f"Bearer {access_token}"}
+    response = requests.get(api_url, headers=headers)
+
+    if response.status_code != 200:
+        raise DataContractException(
+            type="schema",
+            name="Retrieve unity catalog schema",
+            reason=f"Failed to retrieve unity catalog schema from databricks instance: {response.status_code} {response.text}",
+            engine="datacontract",
+        )
+
+    convert_unity_schema(data_contract_specification, response.json())
+
+    return data_contract_specification
+
+
+def convert_unity_schema(
+    data_contract_specification: DataContractSpecification, unity_schema: dict
+) -> DataContractSpecification:
+    if data_contract_specification.models is None:
+        data_contract_specification.models = {}
+
+    fields = import_table_fields(unity_schema.get("columns"))
+
+    table_id = unity_schema.get("table_id")
+
+    data_contract_specification.models[table_id] = Model(fields=fields, type="table")
+
+    if unity_schema.get("name") is not None:
+        data_contract_specification.models[table_id].title = unity_schema.get("name")
+
+    return data_contract_specification
+
+
+def import_table_fields(table_fields):
+    imported_fields = {}
+    for field in table_fields:
+        field_name = field.get("name")
+        imported_fields[field_name] = Field()
+        imported_fields[field_name].required = field.get("nullable") == "false"
+        imported_fields[field_name].description = field.get("comment")
+
+        # databricks api 2.1 specifies that type_name can be any of:
+        # BOOLEAN | BYTE | SHORT | INT | LONG | FLOAT | DOUBLE | DATE | TIMESTAMP | TIMESTAMP_NTZ | STRING
+        # | BINARY | DECIMAL | INTERVAL | ARRAY | STRUCT | MAP | CHAR | NULL | USER_DEFINED_TYPE | TABLE_TYPE
+        if field.get("type_name") in ["INTERVAL", "ARRAY", "STRUCT", "MAP", "USER_DEFINED_TYPE", "TABLE_TYPE"]:
+            # complex types are not supported, yet
+            raise DataContractException(
+                type="schema",
+                result="failed",
+                name="Map unity type to data contract type",
+                reason=f"type ${field.get('type_name')} is not supported yet for unity import",
+                engine="datacontract",
+            )
+
+        imported_fields[field_name].type = map_type_from_unity(field.get("type_name"))
+
+    return imported_fields
+
+
+def map_type_from_unity(type_str: str):
+    if type_str == "BOOLEAN":
+        return "boolean"
+    elif type_str == "BYTE":
+        return "bytes"
+    elif type_str == "SHORT":
+        return "int"
+    elif type_str == "INT":
+        return "int"
+    elif type_str == "LONG":
+        return "long"
+    elif type_str == "FLOAT":
+        return "float"
+    elif type_str == "DOUBLE":
+        return "double"
+    elif type_str == "DATE":
+        return "date"
+    elif type_str == "TIMESTAMP":
+        return "timestamp"
+    elif type_str == "TIMESTAMP_NTZ":
+        return "timestamp_ntz"
+    elif type_str == "STRING":
+        return "string"
+    elif type_str == "BINARY":
+        return "bytes"
+    elif type_str == "DECIMAL":
+        return "decimal"
+    elif type_str == "CHAR":
+        return "varchar"
+    elif type_str == "NULL":
+        return "null"
+    else:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="Map unity type to data contract type",
+            reason=f"Unsupported type {type_str} in unity json definition.",
+            engine="datacontract",
+        )

datacontract/lint/resolve.py

@@ -61,6 +61,10 @@ def inline_definitions_into_data_contract(spec: DataContractSpecification):
             for field_name in field.model_fields.keys():
                 if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
                     setattr(field, field_name, getattr(definition, field_name))
+            # extras
+            for extra_field_name, extra_field_value in definition.model_extra.items():
+                if extra_field_name not in field.model_extra.keys():
+                    setattr(field, extra_field_name, extra_field_value)
 
 
 def resolve_definition_ref(ref, definitions) -> Definition:
@@ -68,7 +72,11 @@ def resolve_definition_ref(ref, definitions) -> Definition:
         definition_str = fetch_resource(ref)
         definition_dict = to_yaml(definition_str)
         return Definition(**definition_dict)
-
+    elif ref.startswith("file://"):
+        path = ref.replace("file://", "")
+        definition_str = fetch_file(path)
+        definition_dict = to_yaml(definition_str)
+        return Definition(**definition_dict)
     elif ref.startswith("#/definitions/"):
         definition_name = ref.split("#/definitions/")[1]
         return definitions[definition_name]
@@ -82,6 +90,19 @@ def resolve_definition_ref(ref, definitions) -> Definition:
         )
 
 
+def fetch_file(path) -> str:
+    if not os.path.exists(path):
+        raise DataContractException(
+            type="export",
+            result="failed",
+            name="Check that data contract definition is valid",
+            reason=f"Cannot resolve reference {path}",
+            engine="datacontract",
+        )
+    with open(path, "r") as file:
+        return file.read()
+
+
 def resolve_quality_ref(quality: Quality):
     """
     Return the content of a ref file path

datacontract/model/data_contract_specification.py

@@ -10,9 +10,15 @@ class Contact(pyd.BaseModel):
     url: str = None
     email: str = None
 
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
+
 
 class Server(pyd.BaseModel):
     type: str = None
+    description: str = None
+    environment: str = None
     format: str = None
     project: str = None
     dataset: str = None
@@ -33,12 +39,21 @@ class Server(pyd.BaseModel):
     outputPortId: str = None
     driver: str = None
 
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
+
 
 class Terms(pyd.BaseModel):
     usage: str = None
     limitations: str = None
     billing: str = None
     noticePeriod: str = None
+    description: str = None
+
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
 
 
 class Definition(pyd.BaseModel):
@@ -59,8 +74,13 @@ class Definition(pyd.BaseModel):
     pii: bool = None
     classification: str = None
     tags: List[str] = []
+    links: Dict[str, str] = {}
     example: str = None
 
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
+
 
 class Field(pyd.BaseModel):
     ref: str = pyd.Field(default=None, alias="$ref")
@@ -84,6 +104,7 @@ class Field(pyd.BaseModel):
     exclusiveMaximum: int = None
     enum: List[str] = []
     tags: List[str] = []
+    links: Dict[str, str] = {}
     fields: Dict[str, "Field"] = {}
     items: "Field" = None
     precision: int = None
@@ -91,13 +112,18 @@ class Field(pyd.BaseModel):
     example: str = None
     config: Dict[str, Any] = None
 
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
+
 
 class Model(pyd.BaseModel):
-    description: str = None
-    type: str = None
-    namespace: str = None
-    title: str = None
+    description: Optional[str] = None
+    type: Optional[str] = None
+    namespace: Optional[str] = None
+    title: Optional[str] = None
     fields: Dict[str, Field] = {}
+    config: Dict[str, Any] = None
 
 
 class Info(pyd.BaseModel):
@@ -108,6 +134,10 @@ class Info(pyd.BaseModel):
     owner: str = None
     contact: Contact = None
 
+    model_config = pyd.ConfigDict(
+        extra="allow",
+    )
+
 
 class Example(pyd.BaseModel):
     type: str = None
@@ -189,6 +219,8 @@ class DataContractSpecification(pyd.BaseModel):
     examples: List[Example] = []
     quality: Quality = None
     servicelevels: Optional[ServiceLevel] = None
+    links: Dict[str, str] = {}
+    tags: List[str] = []
 
     @classmethod
     def from_file(cls, file):

datacontract/templates/datacontract.html

@@ -41,6 +41,21 @@
             <div class="mt-1 flex flex-col sm:mt-0 sm:flex-row sm:flex-wrap sm:space-x-6">
               {{ datacontract.id }}
             </div>
+            <div class="mt-1 flex flex-col sm:mt-0 sm:flex-row sm:flex-wrap sm:space-x-6">
+              {% if datacontract.tags %}
+              <div class="mt-2 flex items-center text-sm text-gray-500 whitespace-nowrap">
+                <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="mr-1.5 h-5 w-5 flex-shrink-0 text-gray-400">
+                  <path stroke-linecap="round" stroke-linejoin="round" d="M9.568 3H5.25A2.25 2.25 0 003 5.25v4.318c0 .597.237 1.17.659 1.591l9.581 9.581c.699.699 1.78.872 2.607.33a18.095 18.095 0 005.223-5.223c.542-.827.369-1.908-.33-2.607L11.16 3.66A2.25 2.25 0 009.568 3z" />
+                  <path stroke-linecap="round" stroke-linejoin="round" d="M6 6h.008v.008H6V6z" />
+                </svg>
+                {% for tag in datacontract.tags %}
+                <span class="inline-flex items-center rounded-full bg-indigo-100 px-2.5 py-0.5 text-xs font-medium text-gray-800 mr-1">
+                  <span>{{ tag }}</span>
+                </span>
+                {% endfor %}
+              </div>
+              {% endif %}
+            </div>
           </div>
           <div class="mt-5 flex lg:mt-0 lg:ml-4 gap-3 items-center">
             <button
@@ -129,14 +144,14 @@
             </div>
             {% endfor %}
           </section>
-          
+
           {% if datacontract.definitions %}
           <section id="definitions">
             <div class="px-4 sm:px-0">
               <h1 class="text-base font-semibold leading-6 text-gray-900">Definitions</h1>
               <p class="text-sm text-gray-500">Domain specific definitions in the data contract</p>
             </div>
-            
+
             {% for definition_name, definition in datacontract.definitions.items() %}
             {{ render_partial('partials/definition.html', definition_name = definition_name, definition = definition)}}
             {% endfor %}

datacontract/templates/partials/datacontract_information.html

@@ -6,6 +6,19 @@
 
   <div class="px-4 py-5 sm:px-6">
 
+    {% if datacontract.links %}
+      <div class="flex flex-wrap gap-3">
+        {% for name, href in datacontract.links.items() %}
+        <a href="{{ href }}" class="flex flex-col text-center rounded-md bg-white px-2 py-2 text-sm font-medium text-gray-500 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50 mb-6" target="_blank" style="min-width: 100px">
+          <div class="mx-auto w-8 h-8 my-2">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M23.91,5.29a3.13,3.13,0,0,0-.72-1.23.35.35,0,0,0-.49,0,.34.34,0,0,0,0,.48,2.66,2.66,0,0,1,.54,1,1.56,1.56,0,0,1-.1,1,19.42,19.42,0,0,1-3.49,3.64,9.62,9.62,0,0,1-2.74,1.8,5.51,5.51,0,0,1-1.71.45.91.91,0,0,1-.73-.3c-.2-.22-.52-.52-.82-.83.66-.65,1.32-1.35,2.08-2.09a.3.3,0,0,0,0-.43.31.31,0,0,0-.43,0c-.84.6-1.6,1.14-2.32,1.69a7.18,7.18,0,0,1-.32-.81,5.23,5.23,0,0,1-.23-1,3.23,3.23,0,0,1,.94-2.5,31.64,31.64,0,0,1,3.65-3,5.24,5.24,0,0,1,1.63-1,2.56,2.56,0,0,1,1.77,0,2.08,2.08,0,0,1,.65.41,2.31,2.31,0,0,1,.49.61.3.3,0,0,0,.54-.26A2.78,2.78,0,0,0,21.52,2a2.91,2.91,0,0,0-.84-.57,3.23,3.23,0,0,0-1.81-.18,5.64,5.64,0,0,0-2.38,1.13,30.94,30.94,0,0,0-3.91,3,4.26,4.26,0,0,0-1.29,3.34,6.18,6.18,0,0,0,.27,1.23A11.36,11.36,0,0,0,12,11.06l0,.07-1.21,1c-.76.67-1.5,1.37-2.38,2.07a.34.34,0,0,0-.09.48.35.35,0,0,0,.48.09c1-.58,1.81-1.14,2.62-1.74.36-.26.71-.54,1-.83l.24-.21.82.82a2,2,0,0,0,1.6.67,6.72,6.72,0,0,0,2-.57,10.52,10.52,0,0,0,3-2.1,19.88,19.88,0,0,0,3.58-4.07A2.3,2.3,0,0,0,23.91,5.29Z" fill="#191919" fill-rule="evenodd"></path><path d="M11.86,14.07a.34.34,0,0,0,0,.48.81.81,0,0,1,.29.57,1.28,1.28,0,0,1-.27.65,29.38,29.38,0,0,1-3.6,3.65,9.07,9.07,0,0,1-2.73,1.79,5.43,5.43,0,0,1-1.71.45,1,1,0,0,1-.73-.31c-.22-.25-.6-.61-.93-1a2.71,2.71,0,0,1-.43-.56,9.71,9.71,0,0,1-.39-1,4.53,4.53,0,0,1-.22-1,3.21,3.21,0,0,1,1-2.51,30.47,30.47,0,0,1,3.66-3,5.14,5.14,0,0,1,1.65-1,2.62,2.62,0,0,1,1.79,0,.3.3,0,0,0,.4-.16.31.31,0,0,0-.17-.4,3.16,3.16,0,0,0-1.79-.19A5.63,5.63,0,0,0,5.27,11.7a30.41,30.41,0,0,0-3.93,2.94A4.21,4.21,0,0,0,0,18a6,6,0,0,0,.19,1A9.53,9.53,0,0,0,.7,20.3a3.53,3.53,0,0,0,.47.69c.36.41.83.84,1.09,1.13a2,2,0,0,0,1.59.68,6.21,6.21,0,0,0,2.05-.56,10,10,0,0,0,3-2.1,30.27,30.27,0,0,0,3.6-4,2,2,0,0,0,.36-1.07,1.46,1.46,0,0,0-.53-1A.35.35,0,0,0,11.86,14.07Z" fill="#191919" fill-rule="evenodd"></path><path d="M6.43,15.15l1.16-.79A.31.31,0,0,0,7.73,14a.3.3,0,0,0-.41-.13L6,14.49a7.39,7.39,0,0,0-2.3,1.83,2.35,2.35,0,0,0-.38,2.26,1,1,0,0,0,1,.68,2.41,2.41,0,0,0,1.17-.43,9.29,9.29,0,0,0,.91-.75c.66-.6,1.25-1.26,1.86-1.85a.35.35,0,0,0,0-.49.36.36,0,0,0-.49,0c-.57.45-1.13.95-1.73,1.42a10.58,10.58,0,0,1-1.17.81c-1.23.74-.36-.89-.3-1A7.43,7.43,0,0,1,6.43,15.15Z" fill="#0c6fff" fill-rule="evenodd"></path><path d="M19,7a6.13,6.13,0,0,1-1.26,1.28c-.21.15-.5.38-.78.56a1.9,1.9,0,0,1-.49.25.34.34,0,0,0-.25.41.34.34,0,0,0,.41.26,2.47,2.47,0,0,0,.55-.2c.35-.18.73-.44,1-.61a6.9,6.9,0,0,0,1.57-1.33A8.69,8.69,0,0,0,20.9,6a3.7,3.7,0,0,0,.43-1.13A1,1,0,0,0,21,4a1.36,1.36,0,0,0-.89-.39,3,3,0,0,0-1.08.17,6.4,6.4,0,0,0-1.63.82,15.32,15.32,0,0,0-2,1.7,4.54,4.54,0,0,0-.65.74,2.69,2.69,0,0,0-.31.61,3.5,3.5,0,0,0-.07.9.33.33,0,0,0,.17.24A.31.31,0,0,0,15,8.63c.18-.26-.17-.6.47-1.3A5.64,5.64,0,0,1,16,6.88a19,19,0,0,1,2-1.42,5.88,5.88,0,0,1,1.39-.61A1.87,1.87,0,0,1,20,4.73c.54,0,.12.31-.08.76A7.29,7.29,0,0,1,19,7Z" fill="#0c6fff" fill-rule="evenodd"></path></g></svg>
+          </div>
+          <div>{{ name }}</div>
+        </a>
+        {% endfor %}
+      </div>
+    {% endif %}
+
     <dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
       <div class="sm:col-span-1">
         <dt class="text-sm font-medium text-gray-500">Title</dt>
@@ -42,6 +55,13 @@
       </div>
       {% endif %}
 
+      {% for key, value in datacontract.info.model_extra.items()  %}
+      <div class="sm:col-span-1">
+        <dt class="text-sm font-medium text-gray-500">{{ key }}</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{ value }}</dd>
+      </div>
+      {% endfor %}
+
       {% if datacontract.info.contact %}
       <div class="sm:col-span-1">
         <dt class="text-sm font-medium text-gray-500">Contact</dt>

datacontract/templates/partials/datacontract_terms.html

@@ -39,6 +39,13 @@
         </div>
       {% endif %}
 
+      {% for key, value in datacontract.terms.model_extra.items()  %}
+      <div class="sm:col-span-1">
+        <dt class="text-sm font-medium text-gray-500">{{ key }}</dt>
+        <dd class="mt-1 text-sm text-gray-900">{{ value }}</dd>
+      </div>
+      {% endfor %}
+
     </dl>
   </div>
 </div>

datacontract/templates/partials/definition.html

@@ -14,7 +14,7 @@
           </tr>
           </thead>
           <tbody class="divide-y divide-gray-200 bg-white">
-            <tr>
+            <tr id="/definitions/{{ definition_name }}">
               <td class="whitespace-nowrap py-2 pl-4 pr-2 text-sm font-medium text-gray-900 sm:pl-6 w-2/12">
                 <div class="py-2 text-sm">
                   {% if definition.title %}
@@ -87,6 +87,14 @@
                   {% if definition.pii %}
                   <span class="inline-flex items-center rounded-md bg-yellow-50 px-1 py-1 text-xs font-medium text-yellow-600 ring-1 ring-inset ring-yellow-500/10 mr-1 mt-1">PII</span>
                   {% endif %}
+                  {% for key, value in definition.model_extra.items()  %}
+                  <span class="inline-flex items-center rounded-md bg-gray-50 px-1 py-1 text-xs font-medium text-gray-600 ring-1 ring-inset ring-gray-500/10 mr-1 mt-1">{{ key }}:{{ value }}</span>
+                  {% endfor %}
+                  {% if definition.links %}
+                    {% for name,href in definition.links.items() %}
+                    <a href="{{ href }}" class="inline-flex items-center px-1 py-1 mr-1 mt-1 text-sky-500 hover:text-gray-700 text-xs font-semibold">{{ name }}</a>
+                    {% endfor %}
+                  {% endif %}
                 </div>
               </td>
 

datacontract/templates/partials/model_field.html

@@ -14,9 +14,11 @@
 {% endif %}
     <div class="py-2 text-sm">
       {% if field.title %}
-      <div>{{ field.title }}</div>
+      <span>{{ field.title }}</span><br>
       {% endif %}
-      <div class="font-mono">{{ field_name }}</div>
+      <span class="font-mono flex">{{ field_name }}{% if field.ref %} <a href="{{ field.ref }}">
+        <svg title="Definition" class="mr-1.5 h-5 w-5 flex-shrink-0" xmlns="http://www.w3.org/2000/svg" viewBox="-0.75 -0.75 24 24"><defs></defs><path d="M3.046875 5.15625h16.40625s0.9375 0 0.9375 0.9375v10.3125s0 0.9375 -0.9375 0.9375H3.046875s-0.9375 0 -0.9375 -0.9375v-10.3125s0 -0.9375 0.9375 -0.9375" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"></path><path d="m12.568125 10.3125 4.6875 0" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"></path><path d="m12.568125 13.125 4.6875 0" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"></path><path d="M5.068124999999999 8.4375h4.6875v4.6875h-4.6875Z" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"></path></svg>
+      </a>{% endif %}</span>
     </div>
   </td>
   <td class="whitespace-nowrap px-1 py-2 text-sm text-gray-500 w-1/12">
@@ -83,15 +85,30 @@
       {% if field.pii %}
       <span class="inline-flex items-center rounded-md bg-yellow-50 px-1 py-1 text-xs font-medium text-yellow-600 ring-1 ring-inset ring-yellow-500/10 mr-1 mt-1">PII</span>
       {% endif %}
+      {% for key, value in field.model_extra.items()  %}
+      <span class="inline-flex items-center rounded-md bg-gray-50 px-1 py-1 text-xs font-medium text-gray-600 ring-1 ring-inset ring-gray-500/10 mr-1 mt-1">{{ key }}:{{ value }}</span>
+      {% endfor %}
+      {% if field.links %}
+      {% for name,href in field.links.items() %}
+      <a href="{{ href }}" class="inline-flex items-center px-1 py-1 mr-1 mt-1 text-sky-500 hover:text-gray-700 text-xs font-semibold">{{ name }}</a>
+      {% endfor %}
+      {% endif %}
     </div>
   </td>
 </tr>
 
+{% macro render_nested_partial(field_name, field, level) %}
+{{ render_partial('partials/model_field.html', nested=True, field_name=field_name, field=field, level=level + 1) }}
+<!-- Mark the end of the contained fields -->
+<tr style="--tw-divide-y-reverse: 2"></tr>
+{% endmacro %}
+
 {% if field.fields %}
 {% for field_name, field in field.fields.items() %}
-  {{ render_partial('partials/model_field.html', nested = True, field_name=field_name, field = field, level = level + 1) }}
+{{ render_nested_partial(field_name, field, level) }}
 {% endfor %}
-<!-- Mark the end of the contained fields -->
-<tr style="--tw-divide-y-reverse: 2">
-</tr>
+{% endif %}
+
+{% if field.items %}
+{{ render_nested_partial("item", field.items, level) }}
 {% endif %}