datacontract-cli 0.10.35__py3-none-any.whl → 0.10.36__py3-none-any.whl

datacontract/export/dqx_converter.py

@@ -61,13 +61,18 @@ def process_quality_rule(rule: Quality, column_name: str) -> Dict[str, Any]:
     specification = rule_data[DqxKeys.SPECIFICATION]
     check = specification[DqxKeys.CHECK]
 
-    arguments = check.setdefault(DqxKeys.ARGUMENTS, {})
-
-    if DqxKeys.COL_NAME not in arguments and DqxKeys.COL_NAMES not in arguments and DqxKeys.COLUMNS not in arguments:
-        if check[DqxKeys.FUNCTION] not in ("is_unique", "foreign_key"):
-            arguments[DqxKeys.COL_NAME] = column_name
-        else:
-            arguments[DqxKeys.COLUMNS] = [column_name]
+    if column_name:
+        arguments = check.setdefault(DqxKeys.ARGUMENTS, {})
+
+        if (
+            DqxKeys.COL_NAME not in arguments
+            and DqxKeys.COL_NAMES not in arguments
+            and DqxKeys.COLUMNS not in arguments
+        ):
+            if check[DqxKeys.FUNCTION] not in ("is_unique", "foreign_key"):
+                arguments[DqxKeys.COL_NAME] = column_name
+            else:
+                arguments[DqxKeys.COLUMNS] = [column_name]
 
     return specification
 

datacontract/export/excel_exporter.py

@@ -283,7 +283,7 @@ def fill_single_property_template(
     sheet: Worksheet, row_index: int, prefix: str, property: SchemaProperty, header_map: dict
 ) -> int:
     """Fill a single property row using the template's column structure"""
-    property_name = f"{prefix}.{property.name}" if prefix else property.name
+    property_name = f"{prefix}{'.' + property.name if property.name else ''}" if prefix else property.name
 
     # Helper function to set cell value by header name
     def set_by_header(header_name: str, value: Any):
@@ -307,7 +307,7 @@ def fill_single_property_template(
     set_by_header("Classification", property.classification)
     set_by_header("Tags", ",".join(property.tags) if property.tags else "")
     set_by_header(
-        "Example(s)", ",".join(property.examples) if property.examples else ""
+        "Example(s)", ",".join(map(str, property.examples)) if property.examples else ""
     )  # Note: using "Example(s)" as in template
     set_by_header("Encrypted Name", property.encryptedName)
     set_by_header(
@@ -404,7 +404,7 @@ def fill_properties_quality(
         if not property.name:
             continue
 
-        full_property_name = f"{prefix}.{property.name}" if prefix else property.name
+        full_property_name = f"{prefix}{'.' + property.name if property.name else ''}" if prefix else property.name
 
         # Add quality attributes for this property
         if property.quality:

datacontract/export/markdown_converter.py

@@ -82,7 +82,7 @@ def obj_attributes_to_markdown(obj: BaseModel, excluded_fields: set = set(), is_
         if value
     ]
     description = f"*{description_to_markdown(description_value)}*"
-    extra = [extra_to_markdown(obj)] if obj.model_extra else []
+    extra = [extra_to_markdown(obj, is_in_table_cell)] if obj.model_extra else []
     return newline_char.join([description] + attributes + extra)
 
 
@@ -293,26 +293,45 @@ def dict_to_markdown(dictionary: Dict[str, str]) -> str:
     return "\n".join(markdown_parts) + "\n"
 
 
-def extra_to_markdown(obj: BaseModel) -> str:
+def extra_to_markdown(obj: BaseModel, is_in_table_cell: bool = False) -> str:
     """
     Convert the extra attributes of a data contract to Markdown format.
     Args:
         obj (BaseModel): The data contract object containing extra attributes.
+        is_in_table_cell (bool): Whether the extra attributes are in a table cell.
     Returns:
         str: A Markdown formatted string representing the extra attributes of the data contract.
     """
-    markdown_part = ""
     extra = obj.model_extra
-    if extra:
-        for key_extra, value_extra in extra.items():
-            markdown_part += f"\n### {key_extra.capitalize()}\n"
-            if isinstance(value_extra, list) and len(value_extra):
-                if isinstance(value_extra[0], dict):
-                    markdown_part += array_of_dict_to_markdown(value_extra)
-                elif isinstance(value_extra[0], str):
-                    markdown_part += array_to_markdown(value_extra)
-            elif isinstance(value_extra, dict):
-                markdown_part += dict_to_markdown(value_extra)
-            else:
-                markdown_part += f"{str(value_extra)}\n"
-    return markdown_part
+
+    if not extra:
+        return ""
+
+    bullet_char = "•"
+    value_line_ending = "" if is_in_table_cell else "\n"
+    row_suffix = "<br>" if is_in_table_cell else ""
+
+    def render_header(key: str) -> str:
+        return f"{bullet_char} **{key}:** " if is_in_table_cell else f"\n### {key.capitalize()}\n"
+
+    parts: list[str] = []
+    for key_extra, value_extra in extra.items():
+        if not value_extra:
+            continue
+
+        parts.append(render_header(key_extra))
+
+        if isinstance(value_extra, list) and len(value_extra):
+            if isinstance(value_extra[0], dict):
+                parts.append(array_of_dict_to_markdown(value_extra))
+            elif isinstance(value_extra[0], str):
+                parts.append(array_to_markdown(value_extra))
+        elif isinstance(value_extra, dict):
+            parts.append(dict_to_markdown(value_extra))
+        else:
+            parts.append(f"{str(value_extra)}{value_line_ending}")
+
+        if row_suffix:
+            parts.append(row_suffix)
+
+    return "".join(parts)

datacontract/export/rdf_converter.py

@@ -57,8 +57,8 @@ def to_rdf(data_contract_spec: DataContractSpecification, base) -> Graph:
     else:
         g = Graph(base=Namespace(""))
 
-    dc = Namespace("https://datacontract.com/DataContractSpecification/1.2.0/")
-    dcx = Namespace("https://datacontract.com/DataContractSpecification/1.2.0/Extension/")
+    dc = Namespace("https://datacontract.com/DataContractSpecification/1.2.1/")
+    dcx = Namespace("https://datacontract.com/DataContractSpecification/1.2.1/Extension/")
 
     g.bind("dc", dc)
     g.bind("dcx", dcx)

datacontract/export/sql_type_converter.py

@@ -133,8 +133,9 @@ def convert_to_dataframe(field: Field) -> None | str:
     if type.lower() in ["time"]:
         return "STRING"
     if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "DECIMAL"
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return f"DECIMAL({precision},{scale})"
     if type.lower() in ["float"]:
         return "FLOAT"
     if type.lower() in ["double"]:
@@ -182,8 +183,9 @@ def convert_to_databricks(field: Field) -> None | str:
     if type.lower() in ["time"]:
         return "STRING"
     if type.lower() in ["number", "decimal", "numeric"]:
-        # precision and scale not supported by data contract
-        return "DECIMAL"
+        precision = field.precision if field.precision is not None else 38
+        scale = field.scale if field.scale is not None else 0
+        return f"DECIMAL({precision},{scale})"
     if type.lower() in ["float"]:
         return "FLOAT"
     if type.lower() in ["double"]:

datacontract/imports/odcs_v3_importer.py

@@ -207,7 +207,11 @@ def import_models(odcs: Any) -> Dict[str, Model]:
         schema_physical_name = odcs_schema.physicalName
         schema_description = odcs_schema.description if odcs_schema.description is not None else ""
         model_name = schema_physical_name if schema_physical_name is not None else schema_name
-        model = Model(description=" ".join(schema_description.splitlines()) if schema_description else "", type="table")
+        model = Model(
+            description=" ".join(schema_description.splitlines()) if schema_description else "",
+            type="table",
+            tags=odcs_schema.tags if odcs_schema.tags is not None else None,
+        )
         model.fields = import_fields(odcs_schema.properties, custom_type_mappings, server_type=get_server_type(odcs))
         if odcs_schema.quality is not None:
             model.quality = convert_quality_list(odcs_schema.quality)
@@ -231,6 +235,8 @@ def convert_quality_list(odcs_quality_list):
                 quality.description = odcs_quality.description
             if odcs_quality.query is not None:
                 quality.query = odcs_quality.query
+            if odcs_quality.rule is not None:
+                quality.metric = odcs_quality.rule
             if odcs_quality.mustBe is not None:
                 quality.mustBe = odcs_quality.mustBe
             if odcs_quality.mustNotBe is not None:
@@ -238,11 +244,11 @@ def convert_quality_list(odcs_quality_list):
             if odcs_quality.mustBeGreaterThan is not None:
                 quality.mustBeGreaterThan = odcs_quality.mustBeGreaterThan
             if odcs_quality.mustBeGreaterOrEqualTo is not None:
-                quality.mustBeGreaterThanOrEqualTo = odcs_quality.mustBeGreaterOrEqualTo
+                quality.mustBeGreaterOrEqualTo = odcs_quality.mustBeGreaterOrEqualTo
             if odcs_quality.mustBeLessThan is not None:
                 quality.mustBeLessThan = odcs_quality.mustBeLessThan
             if odcs_quality.mustBeLessOrEqualTo is not None:
-                quality.mustBeLessThanOrEqualTo = odcs_quality.mustBeLessOrEqualTo
+                quality.mustBeLessOrEqualTo = odcs_quality.mustBeLessOrEqualTo
             if odcs_quality.mustBeBetween is not None:
                 quality.mustBeBetween = odcs_quality.mustBeBetween
             if odcs_quality.mustNotBeBetween is not None:
@@ -255,8 +261,6 @@ def convert_quality_list(odcs_quality_list):
                 quality.model_extra["businessImpact"] = odcs_quality.businessImpact
             if odcs_quality.dimension is not None:
                 quality.model_extra["dimension"] = odcs_quality.dimension
-            if odcs_quality.rule is not None:
-                quality.model_extra["rule"] = odcs_quality.rule
             if odcs_quality.schedule is not None:
                 quality.model_extra["schedule"] = odcs_quality.schedule
             if odcs_quality.scheduler is not None:
@@ -330,7 +334,7 @@ def import_fields(
         return result
 
     for odcs_property in odcs_properties:
-        mapped_type = map_type(odcs_property.logicalType, custom_type_mappings)
+        mapped_type = map_type(odcs_property.logicalType, custom_type_mappings, odcs_property.physicalType)
         if mapped_type is not None:
             property_name = odcs_property.name
             description = odcs_property.description if odcs_property.description is not None else None
@@ -373,23 +377,72 @@ def import_fields(
 
             result[property_name] = field
         else:
-            logger.info(
-                f"Can't map {odcs_property.name} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{odcs_property.logicalType}' that defines your expected type as the 'value'"
+            type_info = f"logicalType={odcs_property.logicalType}, physicalType={odcs_property.physicalType}"
+            logger.warning(
+                f"Can't map field '{odcs_property.name}' ({type_info}) to the Datacontract Mapping types. "
+                f"Both logicalType and physicalType are missing or unmappable. "
+                f"Consider introducing a customProperty 'dc_mapping_<type>' that defines your expected type as the 'value'"
             )
 
     return result
 
 
-def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
-    if odcs_type is None:
-        return None
-    t = odcs_type.lower()
-    if t in DATACONTRACT_TYPES:
-        return t
-    elif custom_mappings.get(t) is not None:
-        return custom_mappings.get(t)
-    else:
-        return None
+def map_type(odcs_logical_type: str, custom_mappings: Dict[str, str], physical_type: str = None) -> str | None:
+    # Try to map logicalType first
+    if odcs_logical_type is not None:
+        t = odcs_logical_type.lower()
+        if t in DATACONTRACT_TYPES:
+            return t
+        elif custom_mappings.get(t) is not None:
+            return custom_mappings.get(t)
+
+    # Fallback to physicalType if logicalType is not mapped
+    if physical_type is not None:
+        pt = physical_type.lower()
+        # Remove parameters from physical type (e.g., VARCHAR(50) -> varchar, DECIMAL(10,2) -> decimal)
+        pt_base = pt.split('(')[0].strip()
+
+        # Try direct mapping of physical type
+        if pt in DATACONTRACT_TYPES:
+            return pt
+        elif pt_base in DATACONTRACT_TYPES:
+            return pt_base
+        elif custom_mappings.get(pt) is not None:
+            return custom_mappings.get(pt)
+        elif custom_mappings.get(pt_base) is not None:
+            return custom_mappings.get(pt_base)
+        # Common physical type mappings
+        elif pt_base in ["varchar", "char", "nvarchar", "nchar", "text", "ntext", "string", "character varying"]:
+            return "string"
+        elif pt_base in ["int", "integer", "smallint", "tinyint", "mediumint", "int2", "int4", "int8"]:
+            return "int"
+        elif pt_base in ["bigint", "long", "int64"]:
+            return "long"
+        elif pt_base in ["float", "real", "float4", "float8"]:
+            return "float"
+        elif pt_base in ["double", "double precision"]:
+            return "double"
+        elif pt_base in ["decimal", "numeric", "number"]:
+            return "decimal"
+        elif pt_base in ["boolean", "bool", "bit"]:
+            return "boolean"
+        elif pt_base in ["timestamp", "datetime", "datetime2", "timestamptz", "timestamp with time zone"]:
+            return "timestamp"
+        elif pt_base in ["date"]:
+            return "date"
+        elif pt_base in ["time"]:
+            return "time"
+        elif pt_base in ["json", "jsonb"]:
+            return "json"
+        elif pt_base in ["array"]:
+            return "array"
+        elif pt_base in ["object", "struct", "record"]:
+            return "object"
+        elif pt_base in ["bytes", "binary", "varbinary", "blob", "bytea"]:
+            return "bytes"
+        else:
+            return None
+    return None
 
 
 def get_custom_type_mappings(odcs_custom_properties: List[CustomProperty]) -> Dict[str, str]:

datacontract/imports/unity_importer.py

@@ -88,23 +88,28 @@ def import_unity_from_api(
     """
     try:
         # print(f"Retrieving Unity Catalog schema for table: {unity_table_full_name}")
+        profile = os.getenv("DATACONTRACT_DATABRICKS_PROFILE")
         host, token = os.getenv("DATACONTRACT_DATABRICKS_SERVER_HOSTNAME"), os.getenv("DATACONTRACT_DATABRICKS_TOKEN")
         # print(f"Databricks host: {host}, token: {'***' if token else 'not set'}")
-        if not host:
-            raise DataContractException(
-                type="configuration",
-                name="Databricks configuration",
-                reason="DATACONTRACT_DATABRICKS_SERVER_HOSTNAME environment variable is not set",
-                engine="datacontract",
-            )
-        if not token:
-            raise DataContractException(
+        exception = DataContractException(
                 type="configuration",
                 name="Databricks configuration",
-                reason="DATACONTRACT_DATABRICKS_TOKEN environment variable is not set",
+                reason="",
                 engine="datacontract",
             )
-        workspace_client = WorkspaceClient(host=host, token=token)
+        if not profile and not host and not token:
+            reason = "Either DATACONTRACT_DATABRICKS_PROFILE or both DATACONTRACT_DATABRICKS_SERVER_HOSTNAME and DATACONTRACT_DATABRICKS_TOKEN environment variables must be set"
+            exception.reason = reason
+            raise exception
+        if token and not host:
+            reason = "DATACONTRACT_DATABRICKS_SERVER_HOSTNAME environment variable is not set"
+            exception.reason = reason
+            raise exception
+        if host and not token:
+            reason = "DATACONTRACT_DATABRICKS_TOKEN environment variable is not set"
+            exception.reason = reason
+            raise exception
+        workspace_client = WorkspaceClient(profile=profile) if profile else WorkspaceClient(host=host, token=token)
     except Exception as e:
         raise DataContractException(
             type="schema",

datacontract/init/init_template.py

@@ -3,7 +3,7 @@ import logging
 
 import requests
 
-DEFAULT_DATA_CONTRACT_INIT_TEMPLATE = "datacontract-1.2.0.init.yaml"
+DEFAULT_DATA_CONTRACT_INIT_TEMPLATE = "datacontract-1.2.1.init.yaml"
 
 
 def get_init_template(location: str = None) -> str:

datacontract/lint/resolve.py

@@ -303,7 +303,7 @@ def _resolve_data_contract_from_str(
         # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
         odcs = parse_odcs_v3_from_str(data_contract_str)
 
-        data_contract_specification = DataContractSpecification(dataContractSpecification="1.2.0")
+        data_contract_specification = DataContractSpecification(dataContractSpecification="1.2.1")
         return import_from_odcs(data_contract_specification, odcs)
 
     logging.info("Importing DCS")

datacontract/lint/schema.py

@@ -8,7 +8,7 @@ import requests
 
 from datacontract.model.exceptions import DataContractException
 
-DEFAULT_DATA_CONTRACT_SCHEMA = "datacontract-1.2.0.schema.json"
+DEFAULT_DATA_CONTRACT_SCHEMA = "datacontract-1.2.1.schema.json"
 
 
 def fetch_schema(location: str = None) -> Dict[str, Any]:

datacontract/schemas/datacontract-1.1.0.init.yaml

@@ -1,4 +1,4 @@
-dataContractSpecification: 1.2.0
+dataContractSpecification: 1.2.1
 id: my-data-contract-id
 info:
   title: My Data Contract

datacontract/schemas/datacontract-1.2.0.init.yaml

@@ -1,4 +1,4 @@
-dataContractSpecification: 1.2.0
+dataContractSpecification: 1.2.1
 id: my-data-contract-id
 info:
   title: My Data Contract

datacontract/schemas/datacontract-1.2.1.init.yaml

@@ -0,0 +1,91 @@
+dataContractSpecification: 1.2.1
+id: my-data-contract-id
+info:
+  title: My Data Contract
+  version: 0.0.1
+#  description:
+#  owner:
+#  contact:
+#    name:
+#    url:
+#    email:
+
+
+### servers
+
+#servers:
+#  production:
+#    type: s3
+#    location: s3://
+#    format: parquet
+#    delimiter: new_line
+
+### terms
+
+#terms:
+#  usage:
+#  limitations:
+#  billing:
+#  noticePeriod:
+
+
+### models
+
+# models:
+#   my_model:
+#     description:
+#     type:
+#     fields:
+#       my_field:
+#         type:
+#         description:
+
+
+### definitions
+
+# definitions:
+#   my_field:
+#     domain:
+#     name:
+#     title:
+#     type:
+#     description:
+#     example:
+#     pii:
+#     classification:
+
+
+### servicelevels
+
+#servicelevels:
+#  availability:
+#    description: The server is available during support hours
+#    percentage: 99.9%
+#  retention:
+#    description: Data is retained for one year because!
+#    period: P1Y
+#    unlimited: false
+#  latency:
+#    description: Data is available within 25 hours after the order was placed
+#    threshold: 25h
+#    sourceTimestampField: orders.order_timestamp
+#    processedTimestampField: orders.processed_timestamp
+#  freshness:
+#    description: The age of the youngest row in a table.
+#    threshold: 25h
+#    timestampField: orders.order_timestamp
+#  frequency:
+#    description: Data is delivered once a day
+#    type: batch # or streaming
+#    interval: daily # for batch, either or cron
+#    cron: 0 0 * * * # for batch, either or interval
+#  support:
+#    description: The data is available during typical business hours at headquarters
+#    time: 9am to 5pm in EST on business days
+#    responseTime: 1h
+#  backup:
+#    description: Data is backed up once a week, every Sunday at 0:00 UTC.
+#    interval: weekly
+#    cron: 0 0 * * 0
+#    recoveryTime: 24 hours
+#    recoveryPoint: 1 week