datacontract-cli 0.10.27__py3-none-any.whl → 0.10.29__py3-none-any.whl

datacontract/imports/unity_importer.py

@@ -1,14 +1,14 @@
 import json
 import os
-from typing import List, Optional
+from typing import List
 
 from databricks.sdk import WorkspaceClient
 from databricks.sdk.service.catalog import ColumnInfo, TableInfo
-from pyspark.sql import types
+from open_data_contract_standard.model import OpenDataContractStandard
 
 from datacontract.imports.importer import Importer
-from datacontract.imports.spark_importer import _field_from_struct_type
-from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
+from datacontract.imports.sql_importer import map_type_from_sql, to_physical_type_key
+from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server
 from datacontract.model.exceptions import DataContractException
 
 
@@ -18,8 +18,11 @@ class UnityImporter(Importer):
     """
 
     def import_source(
-        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
-    ) -> DataContractSpecification:
+        self,
+        data_contract_specification: DataContractSpecification | OpenDataContractStandard,
+        source: str,
+        import_args: dict,
+    ) -> DataContractSpecification | OpenDataContractStandard:
         """
         Import data contract specification from a source.
 
@@ -35,15 +38,14 @@ class UnityImporter(Importer):
         if source is not None:
             data_contract_specification = import_unity_from_json(data_contract_specification, source)
         else:
-            data_contract_specification = import_unity_from_api(
-                data_contract_specification, import_args.get("unity_table_full_name")
-            )
+            unity_table_full_name_list = import_args.get("unity_table_full_name")
+            data_contract_specification = import_unity_from_api(data_contract_specification, unity_table_full_name_list)
         return data_contract_specification
 
 
 def import_unity_from_json(
-    data_contract_specification: DataContractSpecification, source: str
-) -> DataContractSpecification:
+    data_contract_specification: DataContractSpecification | OpenDataContractStandard, source: str
+) -> DataContractSpecification | OpenDataContractStandard:
     """
     Import data contract specification from a JSON file.
 
@@ -71,39 +73,66 @@ def import_unity_from_json(
 
 
 def import_unity_from_api(
-    data_contract_specification: DataContractSpecification, unity_table_full_name: Optional[str] = None
+    data_contract_specification: DataContractSpecification, unity_table_full_name_list: List[str] = None
 ) -> DataContractSpecification:
     """
     Import data contract specification from Unity Catalog API.
 
     :param data_contract_specification: The data contract specification to be imported.
     :type data_contract_specification: DataContractSpecification
-    :param unity_table_full_name: The full name of the Unity table.
-    :type unity_table_full_name: Optional[str]
+    :param unity_table_full_name_list: The full name of the Unity table.
+    :type unity_table_full_name_list: list[str]
     :return: The imported data contract specification.
     :rtype: DataContractSpecification
     :raises DataContractException: If there is an error retrieving the schema from the API.
     """
     try:
-        workspace_client = WorkspaceClient()
-        unity_schema: TableInfo = workspace_client.tables.get(unity_table_full_name)
+        # print(f"Retrieving Unity Catalog schema for table: {unity_table_full_name}")
+        host, token = os.getenv("DATACONTRACT_DATABRICKS_SERVER_HOSTNAME"), os.getenv("DATACONTRACT_DATABRICKS_TOKEN")
+        # print(f"Databricks host: {host}, token: {'***' if token else 'not set'}")
+        if not host:
+            raise DataContractException(
+                type="configuration",
+                name="Databricks configuration",
+                reason="DATACONTRACT_DATABRICKS_SERVER_HOSTNAME environment variable is not set",
+                engine="datacontract",
+            )
+        if not token:
+            raise DataContractException(
+                type="configuration",
+                name="Databricks configuration",
+                reason="DATACONTRACT_DATABRICKS_TOKEN environment variable is not set",
+                engine="datacontract",
+            )
+        workspace_client = WorkspaceClient(host=host, token=token)
     except Exception as e:
         raise DataContractException(
             type="schema",
             name="Retrieve unity catalog schema",
-            reason=f"Failed to retrieve unity catalog schema from databricks profile: {os.getenv('DATABRICKS_CONFIG_PROFILE')}",
+            reason="Failed to connect to unity catalog schema",
             engine="datacontract",
             original_exception=e,
         )
 
-    convert_unity_schema(data_contract_specification, unity_schema)
+    for unity_table_full_name in unity_table_full_name_list:
+        try:
+            unity_schema: TableInfo = workspace_client.tables.get(unity_table_full_name)
+        except Exception as e:
+            raise DataContractException(
+                type="schema",
+                name="Retrieve unity catalog schema",
+                reason=f"Unity table {unity_table_full_name} not found",
+                engine="datacontract",
+                original_exception=e,
+            )
+        data_contract_specification = convert_unity_schema(data_contract_specification, unity_schema)
 
     return data_contract_specification
 
 
 def convert_unity_schema(
-    data_contract_specification: DataContractSpecification, unity_schema: TableInfo
-) -> DataContractSpecification:
+    data_contract_specification: DataContractSpecification | OpenDataContractStandard, unity_schema: TableInfo
+) -> DataContractSpecification | OpenDataContractStandard:
     """
     Convert Unity schema to data contract specification.
 
@@ -117,6 +146,21 @@ def convert_unity_schema(
     if data_contract_specification.models is None:
         data_contract_specification.models = {}
 
+    if data_contract_specification.servers is None:
+        data_contract_specification.servers = {}
+
+    # Configure databricks server with catalog and schema from Unity table info
+    schema_name = unity_schema.schema_name
+    catalog_name = unity_schema.catalog_name
+    if catalog_name and schema_name:
+        server_name = "myserver"  # Default server name
+
+        data_contract_specification.servers[server_name] = Server(
+            type="databricks",
+            catalog=catalog_name,
+            schema=schema_name,
+        )
+
     fields = import_table_fields(unity_schema.columns)
 
     table_id = unity_schema.name or unity_schema.table_id
@@ -149,25 +193,21 @@ def import_table_fields(columns: List[ColumnInfo]) -> dict[str, Field]:
     imported_fields = {}
 
     for column in columns:
-        struct_field: types.StructField = _type_json_to_spark_field(column.type_json)
-        imported_fields[column.name] = _field_from_struct_type(struct_field)
+        imported_fields[column.name] = _to_field(column)
 
     return imported_fields
 
 
-def _type_json_to_spark_field(type_json: str) -> types.StructField:
-    """
-    Parses a JSON string representing a Spark field and returns a StructField object.
+def _to_field(column: ColumnInfo) -> Field:
+    field = Field()
+    if column.type_name is not None:
+        sql_type = str(column.type_text)
+        field.type = map_type_from_sql(sql_type)
+    physical_type_key = to_physical_type_key("databricks")
+    field.config = {
+        physical_type_key: sql_type,
+    }
+    field.required = column.nullable is None or not column.nullable
+    field.description = column.comment if column.comment else None
 
-    The reason we do this is to leverage the Spark JSON schema parser to handle the
-    complexity of the Spark field types. The field `type_json` in the Unity API is
-    the output of a `StructField.jsonValue()` call.
-
-    :param type_json: The JSON string representing the Spark field.
-    :type type_json: str
-
-    :return: The StructField object.
-    :rtype: types.StructField
-    """
-    type_dict = json.loads(type_json)
-    return types.StructField.fromJson(type_dict)
+    return field

datacontract/init/init_template.py

@@ -3,7 +3,7 @@ import logging
 
 import requests
 
-DEFAULT_DATA_CONTRACT_INIT_TEMPLATE = "datacontract-1.1.0.init.yaml"
+DEFAULT_DATA_CONTRACT_INIT_TEMPLATE = "datacontract-1.2.0.init.yaml"
 
 
 def get_init_template(location: str = None) -> str:

datacontract/integration/datamesh_manager.py

@@ -4,6 +4,9 @@ import requests
 
 from datacontract.model.run import Run
 
+# used to retrieve the HTML location of the published data contract or test results
+RESPONSE_HEADER_LOCATION_HTML = "location-html"
+
 
 def publish_test_results_to_datamesh_manager(run: Run, publish_url: str, ssl_verification: bool):
     try:
@@ -38,7 +41,12 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str, ssl_ver
         if response.status_code != 200:
             run.log_error(f"Error publishing test results to Data Mesh Manager: {response.text}")
             return
-        run.log_info(f"Published test results to {url}")
+        run.log_info("Published test results successfully")
+
+        location_html = response.headers.get(RESPONSE_HEADER_LOCATION_HTML)
+        if location_html is not None and len(location_html) > 0:
+            print(f"🚀 Open {location_html}")
+
     except Exception as e:
         run.log_error(f"Failed publishing test results. Error: {str(e)}")
 
@@ -67,6 +75,12 @@ def publish_data_contract_to_datamesh_manager(data_contract_dict: dict, ssl_veri
         if response.status_code != 200:
             print(f"Error publishing data contract to Data Mesh Manager: {response.text}")
             exit(1)
-        print(f"Published data contract to {url}")
+
+        print("✅ Published data contract successfully")
+
+        location_html = response.headers.get(RESPONSE_HEADER_LOCATION_HTML)
+        if location_html is not None and len(location_html) > 0:
+            print(f"🚀 Open {location_html}")
+
     except Exception as e:
         print(f"Failed publishing data contract. Error: {str(e)}")

datacontract/lint/resolve.py

@@ -5,8 +5,9 @@ import warnings
 import fastjsonschema
 import yaml
 from fastjsonschema import JsonSchemaValueException
+from open_data_contract_standard.model import OpenDataContractStandard
 
-from datacontract.imports.odcs_v3_importer import import_odcs_v3_from_str
+from datacontract.imports.odcs_v3_importer import import_from_odcs, parse_odcs_v3_from_str
 from datacontract.lint.resources import read_resource
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
@@ -46,6 +47,34 @@ def resolve_data_contract(
         )
 
 
+def resolve_data_contract_v2(
+    data_contract_location: str = None,
+    data_contract_str: str = None,
+    data_contract: DataContractSpecification | OpenDataContractStandard = None,
+    schema_location: str = None,
+    inline_definitions: bool = False,
+    inline_quality: bool = False,
+) -> DataContractSpecification | OpenDataContractStandard:
+    if data_contract_location is not None:
+        return resolve_data_contract_from_location_v2(
+            data_contract_location, schema_location, inline_definitions, inline_quality
+        )
+    elif data_contract_str is not None:
+        return _resolve_data_contract_from_str_v2(
+            data_contract_str, schema_location, inline_definitions, inline_quality
+        )
+    elif data_contract is not None:
+        return data_contract
+    else:
+        raise DataContractException(
+            type="lint",
+            result=ResultEnum.failed,
+            name="Check that data contract YAML is valid",
+            reason="Data contract needs to be provided",
+            engine="datacontract",
+        )
+
+
 def resolve_data_contract_dict(
     data_contract_location: str = None,
     data_contract_str: str = None,
@@ -67,6 +96,13 @@ def resolve_data_contract_dict(
         )
 
 
+def resolve_data_contract_from_location_v2(
+    location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
+) -> DataContractSpecification | OpenDataContractStandard:
+    data_contract_str = read_resource(location)
+    return _resolve_data_contract_from_str_v2(data_contract_str, schema_location, inline_definitions, inline_quality)
+
+
 def resolve_data_contract_from_location(
     location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
@@ -242,6 +278,21 @@ def _get_quality_ref_file(quality_spec: str | object) -> str | object:
     return quality_spec
 
 
+def _resolve_data_contract_from_str_v2(
+    data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
+) -> DataContractSpecification | OpenDataContractStandard:
+    yaml_dict = _to_yaml(data_contract_str)
+
+    if is_open_data_contract_standard(yaml_dict):
+        logging.info("Importing ODCS v3")
+        # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
+        odcs = parse_odcs_v3_from_str(data_contract_str)
+        return odcs
+
+    logging.info("Importing DCS")
+    return _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict)
+
+
 def _resolve_data_contract_from_str(
     data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
@@ -250,15 +301,19 @@ def _resolve_data_contract_from_str(
     if is_open_data_contract_standard(yaml_dict):
         logging.info("Importing ODCS v3")
         # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
-        data_contract_specification = DataContractSpecification(dataContractSpecification="1.1.0")
-        return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str)
-    else:
-        logging.info("Importing DCS")
+        odcs = parse_odcs_v3_from_str(data_contract_str)
 
+        data_contract_specification = DataContractSpecification(dataContractSpecification="1.2.0")
+        return import_from_odcs(data_contract_specification, odcs)
+
+    logging.info("Importing DCS")
+    return _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict)
+
+
+def _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict):
     _validate_data_contract_specification_schema(yaml_dict, schema_location)
     data_contract_specification = yaml_dict
     spec = DataContractSpecification(**data_contract_specification)
-
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
     ## Suppress DeprecationWarning when accessing spec.quality,
@@ -276,7 +331,6 @@ def _resolve_data_contract_from_str(
             )
     if spec_quality and inline_quality:
         _resolve_quality_ref(spec_quality)
-
     return spec
 
 

datacontract/lint/schema.py

@@ -8,7 +8,7 @@ import requests
 
 from datacontract.model.exceptions import DataContractException
 
-DEFAULT_DATA_CONTRACT_SCHEMA = "datacontract-1.1.0.schema.json"
+DEFAULT_DATA_CONTRACT_SCHEMA = "datacontract-1.2.0.schema.json"
 
 
 def fetch_schema(location: str = None) -> Dict[str, Any]:

datacontract/schemas/datacontract-1.1.0.init.yaml

@@ -1,4 +1,4 @@
-dataContractSpecification: 1.1.0
+dataContractSpecification: 1.2.0
 id: my-data-contract-id
 info:
   title: My Data Contract

datacontract/schemas/datacontract-1.2.0.init.yaml

@@ -0,0 +1,91 @@
+dataContractSpecification: 1.2.0
+id: my-data-contract-id
+info:
+  title: My Data Contract
+  version: 0.0.1
+#  description:
+#  owner:
+#  contact:
+#    name:
+#    url:
+#    email:
+
+
+### servers
+
+#servers:
+#  production:
+#    type: s3
+#    location: s3://
+#    format: parquet
+#    delimiter: new_line
+
+### terms
+
+#terms:
+#  usage:
+#  limitations:
+#  billing:
+#  noticePeriod:
+
+
+### models
+
+# models:
+#   my_model:
+#     description:
+#     type:
+#     fields:
+#       my_field:
+#         type:
+#         description:
+
+
+### definitions
+
+# definitions:
+#   my_field:
+#     domain:
+#     name:
+#     title:
+#     type:
+#     description:
+#     example:
+#     pii:
+#     classification:
+
+
+### servicelevels
+
+#servicelevels:
+#  availability:
+#    description: The server is available during support hours
+#    percentage: 99.9%
+#  retention:
+#    description: Data is retained for one year because!
+#    period: P1Y
+#    unlimited: false
+#  latency:
+#    description: Data is available within 25 hours after the order was placed
+#    threshold: 25h
+#    sourceTimestampField: orders.order_timestamp
+#    processedTimestampField: orders.processed_timestamp
+#  freshness:
+#    description: The age of the youngest row in a table.
+#    threshold: 25h
+#    timestampField: orders.order_timestamp
+#  frequency:
+#    description: Data is delivered once a day
+#    type: batch # or streaming
+#    interval: daily # for batch, either or cron
+#    cron: 0 0 * * * # for batch, either or interval
+#  support:
+#    description: The data is available during typical business hours at headquarters
+#    time: 9am to 5pm in EST on business days
+#    responseTime: 1h
+#  backup:
+#    description: Data is backed up once a week, every Sunday at 0:00 UTC.
+#    interval: weekly
+#    cron: 0 0 * * 0
+#    recoveryTime: 24 hours
+#    recoveryPoint: 1 week