datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/lint/resolve.py

@@ -1,16 +1,26 @@
+import importlib.resources as resources
 import logging
 import os
+import warnings
+from pathlib import Path
 
 import fastjsonschema
 import yaml
 from fastjsonschema import JsonSchemaValueException
+from open_data_contract_standard.model import OpenDataContractStandard
 
-from datacontract.lint.files import read_file
+from datacontract.imports.odcs_v3_importer import import_from_odcs, parse_odcs_v3_from_str
+from datacontract.lint.resources import read_resource
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Definition, Quality
+from datacontract.model.data_contract_specification import (
+    DataContractSpecification,
+    Definition,
+    DeprecatedQuality,
+)
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.odcs import is_open_data_contract_standard, is_open_data_product_standard
+from datacontract.model.run import ResultEnum
 
 
 def resolve_data_contract(
@@ -19,68 +29,222 @@ def resolve_data_contract(
     data_contract: DataContractSpecification = None,
     schema_location: str = None,
     inline_definitions: bool = False,
+    inline_quality: bool = False,
 ) -> DataContractSpecification:
     if data_contract_location is not None:
-        return resolve_data_contract_from_location(data_contract_location, schema_location, inline_definitions)
+        return resolve_data_contract_from_location(
+            data_contract_location, schema_location, inline_definitions, inline_quality
+        )
     elif data_contract_str is not None:
-        return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions)
+        return _resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
     elif data_contract is not None:
         return data_contract
     else:
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
+            name="Check that data contract YAML is valid",
+            reason="Data contract needs to be provided",
+            engine="datacontract",
+        )
+
+
+def resolve_data_contract_v2(
+    data_contract_location: str = None,
+    data_contract_str: str = None,
+    data_contract: DataContractSpecification | OpenDataContractStandard = None,
+    schema_location: str = None,
+    inline_definitions: bool = False,
+    inline_quality: bool = False,
+) -> DataContractSpecification | OpenDataContractStandard:
+    if data_contract_location is not None:
+        return resolve_data_contract_from_location_v2(
+            data_contract_location, schema_location, inline_definitions, inline_quality
+        )
+    elif data_contract_str is not None:
+        return _resolve_data_contract_from_str_v2(
+            data_contract_str, schema_location, inline_definitions, inline_quality
+        )
+    elif data_contract is not None:
+        return data_contract
+    else:
+        raise DataContractException(
+            type="lint",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason="Data contract needs to be provided",
             engine="datacontract",
         )
 
 
+def resolve_data_contract_dict(
+    data_contract_location: str = None,
+    data_contract_str: str = None,
+    data_contract: DataContractSpecification = None,
+) -> dict:
+    if data_contract_location is not None:
+        return _to_yaml(read_resource(data_contract_location))
+    elif data_contract_str is not None:
+        return _to_yaml(data_contract_str)
+    elif data_contract is not None:
+        return data_contract.model_dump()
+    else:
+        raise DataContractException(
+            type="lint",
+            result=ResultEnum.failed,
+            name="Check that data contract YAML is valid",
+            reason="Data contract needs to be provided",
+            engine="datacontract",
+        )
+
+
+def resolve_data_contract_from_location_v2(
+    location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
+) -> DataContractSpecification | OpenDataContractStandard:
+    data_contract_str = read_resource(location)
+    return _resolve_data_contract_from_str_v2(data_contract_str, schema_location, inline_definitions, inline_quality)
+
+
 def resolve_data_contract_from_location(
-    location, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = True
+    location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
-    if location.startswith("http://") or location.startswith("https://"):
-        data_contract_str = fetch_resource(location)
-    else:
-        data_contract_str = read_file(location)
-    return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, include_quality)
+    data_contract_str = read_resource(location)
+    return _resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
 
 
 def inline_definitions_into_data_contract(spec: DataContractSpecification):
     for model in spec.models.values():
         for field in model.fields.values():
-            # If ref_obj is not empty, we've already inlined definitions.
-            if not field.ref and not field.ref_obj:
-                continue
+            inline_definition_into_field(field, spec)
+
+
+def inline_definition_into_field(field, spec):
+    # iterate recursively over arrays
+    if field.items is not None:
+        inline_definition_into_field(field.items, spec)
 
-            definition = resolve_definition_ref(field.ref, spec.definitions)
-            field.ref_obj = definition
+    # iterate recursively over nested fields
+    if field.fields is not None:
+        for nested_field_name, nested_field in field.fields.items():
+            inline_definition_into_field(nested_field, spec)
 
-            for field_name in field.model_fields.keys():
-                if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
-                    setattr(field, field_name, getattr(definition, field_name))
+    if not field.ref:
+        return
 
+    definition = _resolve_definition_ref(field.ref, spec)
+    for field_name in field.model_fields.keys():
+        if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
+            setattr(field, field_name, getattr(definition, field_name))
+    # extras
+    for extra_field_name, extra_field_value in definition.model_extra.items():
+        if extra_field_name not in field.model_extra.keys():
+            setattr(field, extra_field_name, extra_field_value)
 
-def resolve_definition_ref(ref, definitions) -> Definition:
-    if ref.startswith("http://") or ref.startswith("https://"):
-        definition_str = fetch_resource(ref)
-        definition_dict = to_yaml(definition_str)
-        return Definition(**definition_dict)
 
-    elif ref.startswith("#/definitions/"):
-        definition_name = ref.split("#/definitions/")[1]
-        return definitions[definition_name]
+def _resolve_definition_ref(ref, spec) -> Definition:
+    logging.info(f"Resolving definition ref {ref}")
+
+    if "#" in ref:
+        path, definition_path = ref.split("#")
+    else:
+        path, definition_path = ref, None
+
+    if path.startswith("http://") or path.startswith("https://"):
+        logging.info(f"Resolving definition url {path}")
+
+        definition_str = fetch_resource(path)
+        definition_dict = _to_yaml(definition_str)
+        definition = Definition(**definition_dict)
+        if definition_path is not None:
+            return _find_by_path_in_definition(definition_path, definition)
+        else:
+            return definition
+    elif path.startswith("file://"):
+        logging.info(f"Resolving definition file path {path}")
+
+        path = path.replace("file://", "")
+        definition_str = _fetch_file(path)
+        definition_dict = _to_yaml(definition_str)
+        if definition_path:
+            path_parts = [part for part in definition_path.split("/") if part != ""]
+            for path_part in path_parts:
+                definition_dict = definition_dict.get(path_part, None)
+                if not definition_dict:
+                    raise DataContractException(
+                        type="lint",
+                        result="failed",
+                        name="Check that data contract YAML is valid",
+                        reason=f"Cannot resolve definition {definition_path}, {path_part} not found",
+                        engine="datacontract",
+                    )
+        # this assumes that definitions_dict is a definitions dict, however,
+        # all we know is that it is a file!
+        definition = Definition(**definition_dict)
+        # if definition_path is not None:
+        #     definition = _find_by_path_in_definition(definition_path, definition)
+        return definition
+    elif ref.startswith("#"):
+        logging.info(f"Resolving definition local path {path}")
+
+        definition_path = ref[1:]
+
+        return _find_by_path_in_spec(definition_path, spec)
     else:
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason=f"Cannot resolve reference {ref}",
             engine="datacontract",
         )
 
 
-def resolve_quality_ref(quality: Quality):
+def _find_by_path_in_spec(definition_path: str, spec: DataContractSpecification):
+    path_elements = definition_path.split("/")
+    definition_key = path_elements[2]
+    if definition_key not in spec.definitions:
+        raise DataContractException(
+            type="lint",
+            result=ResultEnum.failed,
+            name="Check that data contract YAML is valid",
+            reason=f"Cannot resolve definition {definition_key}",
+            engine="datacontract",
+        )
+    definition = spec.definitions[definition_key]
+    definition = _find_subfield_in_definition(definition, path_elements[3:])
+    return definition
+
+
+def _find_by_path_in_definition(definition_path: str, definition: Definition):
+    if definition_path == "" or definition_path == "/":
+        return definition
+
+    path_elements = definition_path.split("/")
+    return _find_subfield_in_definition(definition, path_elements[1:])
+
+
+def _find_subfield_in_definition(definition: Definition, path_elements):
+    while len(path_elements) > 0 and path_elements[0] == "fields":
+        definition = definition.fields[path_elements[1]]
+        path_elements = path_elements[2:]
+
+    return definition
+
+
+def _fetch_file(path) -> str:
+    if not os.path.exists(path):
+        raise DataContractException(
+            type="export",
+            result=ResultEnum.failed,
+            name="Check that data contract definition is valid",
+            reason=f"Cannot resolve reference {path}",
+            engine="datacontract",
+        )
+    with open(path, "r") as file:
+        return file.read()
+
+
+def _resolve_quality_ref(quality: DeprecatedQuality):
     """
     Return the content of a ref file path
     @param quality data contract quality specification
@@ -89,13 +253,13 @@ def resolve_quality_ref(quality: Quality):
         specification = quality.specification
         if quality.type == "great-expectations":
             for model, model_quality in specification.items():
-                specification[model] = get_quality_ref_file(model_quality)
+                specification[model] = _get_quality_ref_file(model_quality)
         else:
             if "$ref" in specification:
-                quality.specification = get_quality_ref_file(specification)
+                quality.specification = _get_quality_ref_file(specification)
 
 
-def get_quality_ref_file(quality_spec: str | object) -> str | object:
+def _get_quality_ref_file(quality_spec: str | object) -> str | object:
     """
     Get the file associated with a quality reference
     @param quality_spec quality specification
@@ -106,7 +270,7 @@ def get_quality_ref_file(quality_spec: str | object) -> str | object:
         if not os.path.exists(ref):
             raise DataContractException(
                 type="export",
-                result="failed",
+                result=ResultEnum.failed,
                 name="Check that data contract quality is valid",
                 reason=f"Cannot resolve reference {ref}",
                 engine="datacontract",
@@ -116,26 +280,83 @@ def get_quality_ref_file(quality_spec: str | object) -> str | object:
     return quality_spec
 
 
-def resolve_data_contract_from_str(
-    data_contract_str, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = False
+def _resolve_data_contract_from_str_v2(
+    data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
+) -> DataContractSpecification | OpenDataContractStandard:
+    yaml_dict = _to_yaml(data_contract_str)
+
+    if is_open_data_product_standard(yaml_dict):
+        logging.info("Cannot import ODPS, as not supported")
+        raise DataContractException(
+            type="schema",
+            result=ResultEnum.failed,
+            name="Parse ODCS contract",
+            reason="Cannot parse ODPS product",
+            engine="datacontract",
+        )
+
+    if is_open_data_contract_standard(yaml_dict):
+        logging.info("Importing ODCS v3")
+        # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
+        odcs = parse_odcs_v3_from_str(data_contract_str)
+        return odcs
+
+    logging.info("Importing DCS")
+    return _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict)
+
+
+def _resolve_data_contract_from_str(
+    data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
 ) -> DataContractSpecification:
-    data_contract_yaml_dict = to_yaml(data_contract_str)
-    validate(data_contract_yaml_dict, schema_location)
+    yaml_dict = _to_yaml(data_contract_str)
+
+    if schema_location is None:
+        if is_open_data_contract_standard(yaml_dict):
+            logging.info("Using ODCS 3.0.2 schema to validate data contract")
+            # TODO refactor this to a specific function
+            schema_location = resources.files("datacontract").joinpath("schemas", "odcs-3.0.2.schema.json")
+
+    _validate_json_schema(yaml_dict, schema_location)
+
+    if is_open_data_contract_standard(yaml_dict):
+        logging.info("Importing ODCS v3")
+        # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly
+        odcs = parse_odcs_v3_from_str(data_contract_str)
+
+        data_contract_specification = DataContractSpecification(dataContractSpecification="1.2.1")
+        return import_from_odcs(data_contract_specification, odcs)
 
-    spec = DataContractSpecification(**data_contract_yaml_dict)
+    logging.info("Importing DCS")
+    return _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict)
 
+
+def _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict):
+    _validate_json_schema(yaml_dict, schema_location)
+    data_contract_specification = yaml_dict
+    spec = DataContractSpecification(**data_contract_specification)
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
-    if spec.quality and include_quality:
-        resolve_quality_ref(spec.quality)
-
+    ## Suppress DeprecationWarning when accessing spec.quality,
+    ## iif it is in fact *not* used.
+    with warnings.catch_warnings(record=True) as recorded_warnings:
+        spec_quality = spec.quality
+    for w in recorded_warnings:
+        if not issubclass(w.category, DeprecationWarning) or spec_quality is not None:
+            warnings.warn_explicit(
+                message=w.message,
+                category=w.category,
+                filename=w.filename,
+                lineno=w.lineno,
+                source=w.source,
+            )
+    if spec_quality and inline_quality:
+        _resolve_quality_ref(spec_quality)
     return spec
 
 
-def to_yaml(data_contract_str):
+def _to_yaml(data_contract_str) -> dict:
     try:
-        yaml_dict = yaml.safe_load(data_contract_str)
-        return yaml_dict
+        return yaml.safe_load(data_contract_str)
     except Exception as e:
         logging.warning(f"Cannot parse YAML. Error: {str(e)}")
         raise DataContractException(
@@ -147,16 +368,17 @@ def to_yaml(data_contract_str):
         )
 
 
-def validate(data_contract_yaml, schema_location: str = None):
+def _validate_json_schema(yaml_str, schema_location: str | Path = None):
+    logging.debug(f"Linting data contract with schema at {schema_location}")
     schema = fetch_schema(schema_location)
     try:
-        fastjsonschema.validate(schema, data_contract_yaml)
+        fastjsonschema.validate(schema, yaml_str, use_default=False)
         logging.debug("YAML data is valid.")
     except JsonSchemaValueException as e:
         logging.warning(f"Data Contract YAML is invalid. Validation error: {e.message}")
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason=e.message,
             engine="datacontract",
@@ -165,7 +387,7 @@ def validate(data_contract_yaml, schema_location: str = None):
         logging.warning(f"Data Contract YAML is invalid. Validation error: {str(e)}")
         raise DataContractException(
             type="lint",
-            result="failed",
+            result=ResultEnum.failed,
             name="Check that data contract YAML is valid",
             reason=str(e),
             engine="datacontract",

datacontract/lint/resources.py

@@ -0,0 +1,21 @@
+from datacontract.lint.files import read_file
+from datacontract.lint.urls import fetch_resource
+
+
+def read_resource(location: str) -> str:
+    """
+    Read a resource from a given location.
+
+    If the location is a URL, fetch the resource from the web. API-Keys are supported.
+    Otherwise, read the resource from a local file.
+
+    Args:
+        location (str): The location of the resource, either a URL or a file path.
+
+    Returns:
+        str: The content of the resource.
+    """
+    if location.startswith("http://") or location.startswith("https://"):
+        return fetch_resource(location)
+    else:
+        return read_file(location)

datacontract/lint/schema.py

@@ -1,27 +1,63 @@
+import importlib.resources as resources
 import json
+import logging
 import os
+from pathlib import Path
+from typing import Any, Dict
 
 import requests
 
 from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum
 
+DEFAULT_DATA_CONTRACT_SCHEMA = "datacontract-1.2.1.schema.json"
 
-def fetch_schema(location: str = None):
-    if location is None:
-        location = "https://datacontract.com/datacontract.schema.json"
 
-    if location.startswith("http://") or location.startswith("https://"):
-        response = requests.get(location)
-        return response.json()
+def fetch_schema(location: str | Path = None) -> Dict[str, Any]:
+    """
+    Fetch and return a JSON schema from a given location.
+
+    This function retrieves a JSON schema either from a URL or a local file path.
+    If no location is provided, it defaults to the DataContract schema URL.
+
+    Args:
+        location: The URL or file path of the schema.
+
+    Returns:
+        The JSON schema as a dictionary.
+
+    Raises:
+        DataContractException: If the specified local file does not exist.
+        requests.RequestException: If there's an error fetching the schema from a URL.
+        json.JSONDecodeError: If there's an error decoding the JSON schema.
+
+    """
+    if location is None:
+        logging.info("Use default bundled schema " + DEFAULT_DATA_CONTRACT_SCHEMA)
+        schemas = resources.files("datacontract")
+        schema_file = schemas.joinpath("schemas", DEFAULT_DATA_CONTRACT_SCHEMA)
+        with schema_file.open("r") as file:
+            schema = json.load(file)
     else:
-        if not os.path.exists(location):
-            raise DataContractException(
-                type="lint",
-                name=f"Reading schema from {location}",
-                reason=f"The file '{location}' does not exist.",
-                engine="datacontract",
-                result="error",
-            )
-        with open(location, "r") as file:
-            file_content = file.read()
-        return json.loads(file_content)
+        # Convert Path objects to strings for string operations
+        location_str = str(location)
+
+        if location_str.startswith("http://") or location_str.startswith("https://"):
+            logging.debug(f"Downloading schema from {location_str}")
+            response = requests.get(location_str)
+            schema = response.json()
+        else:
+            if not os.path.exists(location):
+                raise DataContractException(
+                    type="lint",
+                    name=f"Reading schema from {location}",
+                    reason=f"The file '{location}' does not exist.",
+                    engine="datacontract",
+                    result=ResultEnum.error,
+                )
+
+            logging.debug(f"Loading JSON schema locally at {location}")
+            with open(location, "r") as file:
+                schema = json.load(file)
+
+    return schema

datacontract/lint/urls.py

@@ -1,4 +1,5 @@
 import os
+from urllib.parse import urlparse
 
 import requests
 
@@ -25,16 +26,35 @@ def fetch_resource(url: str):
 
 
 def _set_api_key(headers, url):
-    if ".datamesh-manager.com/" not in url:
-        return
+    hostname = urlparse(url).hostname
+
     datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
-    if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
-        print("Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
-        raise DataContractException(
-            type="lint",
-            name=f"Reading data contract from {url}",
-            reason="Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
-            engine="datacontract",
-            result="error",
-        )
-    headers["x-api-key"] = datamesh_manager_api_key
+    datacontract_manager_api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
+
+    if hostname == "datamesh-manager.com" or hostname.endswith(".datamesh-manager.com"):
+        if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
+            print("Error: Data Mesh Manager API key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
+            raise DataContractException(
+                type="lint",
+                name=f"Reading data contract from {url}",
+                reason="Error: Data Mesh Manager API key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
+                engine="datacontract",
+                result="error",
+            )
+        headers["x-api-key"] = datamesh_manager_api_key
+    elif hostname == "datacontract-manager.com" or hostname.endswith(".datacontract-manager.com"):
+        if datacontract_manager_api_key is None or datacontract_manager_api_key == "":
+            print("Error: Data Contract Manager API key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.")
+            raise DataContractException(
+                type="lint",
+                name=f"Reading data contract from {url}",
+                reason="Error: Data Contract Manager API key is not set. Set env variable DATACONTRACT_MANAGER_API_KEY.",
+                engine="datacontract",
+                result="error",
+            )
+        headers["x-api-key"] = datacontract_manager_api_key
+
+    if datamesh_manager_api_key is not None and datamesh_manager_api_key != "":
+        headers["x-api-key"] = datamesh_manager_api_key
+    if datacontract_manager_api_key is not None and datacontract_manager_api_key != "":
+        headers["x-api-key"] = datacontract_manager_api_key

datacontract/model/data_contract_specification/__init__.py

@@ -0,0 +1 @@
+from datacontract_specification.model import *

datacontract/model/exceptions.py

@@ -1,3 +1,6 @@
+from datacontract.model.run import ResultEnum
+
+
 class DataContractException(Exception):
     """Exception raised for errors in the execution of a run.
 
@@ -19,7 +22,7 @@ class DataContractException(Exception):
         engine="datacontract",
         model=None,
         original_exception=None,
-        result: str = "failed",
+        result: ResultEnum = ResultEnum.failed,
         message="Run operation failed",
     ):
         self.type = type

datacontract/model/odcs.py

@@ -0,0 +1,24 @@
+def is_open_data_contract_standard(odcs: dict) -> bool:
+    """
+    Check if the given dictionary is an OpenDataContractStandard.
+
+    Args:
+        odcs (dict): The dictionary to check.
+
+    Returns:
+        bool: True if the dictionary is an OpenDataContractStandard, False otherwise.
+    """
+    return odcs.get("kind") == "DataContract" and odcs.get("apiVersion", "").startswith("v3")
+
+
+def is_open_data_product_standard(odcs: dict) -> bool:
+    """
+    Check if the given dictionary is an open data product standard.
+
+    Args:
+        odcs (dict): The dictionary to check.
+
+    Returns:
+        bool: True if the dictionary is an open data product standard, False otherwise.
+    """
+    return odcs.get("kind") == "DataProduct" and odcs.get("apiVersion", "").startswith("v1")