datacontract-cli 0.9.7__py3-none-any.whl → 0.9.9__py3-none-any.whl

datacontract/lint/linters/notice_period_linter.py

@@ -1,7 +1,9 @@
 import re
 
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification
 from ..lint import Linter, LinterResult
-from datacontract.model.data_contract_specification import DataContractSpecification
+
 
 class NoticePeriodLinter(Linter):
     @property
@@ -25,17 +27,13 @@ class NoticePeriodLinter(Linter):
           (:?[0-9\.,]+M)? # Number of minutes
           (:?[0-9\.,]+S)? # Number of seconds
         )?
-        """, re.VERBOSE
-        )
-    datetime_basic = re.compile(
-        r"P\d{8}T\d{6}")
-    datetime_extended = re.compile(
-        r"P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}")
+        """,
+        re.VERBOSE,
+    )
+    datetime_basic = re.compile(r"P\d{8}T\d{6}")
+    datetime_extended = re.compile(r"P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}")
 
-    def lint_implementation(
-        self,
-        contract: DataContractSpecification
-    ) -> LinterResult:
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
         """Check whether the notice period is specified using ISO8601 duration syntax."""
         if not contract.terms:
             return LinterResult.cautious("No terms defined.")
@@ -43,17 +41,15 @@ class NoticePeriodLinter(Linter):
         if not period:
             return LinterResult.cautious("No notice period defined.")
         if not period.startswith("P"):
-            return LinterResult.erroneous(
-                f"Notice period '{period}' is not a valid"
-                 "ISO8601 duration.")
+            return LinterResult.erroneous(f"Notice period '{period}' is not a valid" "ISO8601 duration.")
         if period == "P":
             return LinterResult.erroneous(
-                f"Notice period 'P' is not a valid"
-                 "ISO8601 duration, requires at least one"
-                 "duration to be specified.")
-        if (not self.simple.fullmatch(period) and
-            not self.datetime_basic.fullmatch(period) and
-            not self.datetime_extended.fullmatch(period)):
-            return LinterResult.erroneous(
-                f"Notice period '{period}' is not a valid ISO8601 duration.")
+                "Notice period 'P' is not a valid" "ISO8601 duration, requires at least one" "duration to be specified."
+            )
+        if (
+            not self.simple.fullmatch(period)
+            and not self.datetime_basic.fullmatch(period)
+            and not self.datetime_extended.fullmatch(period)
+        ):
+            return LinterResult.erroneous(f"Notice period '{period}' is not a valid ISO8601 duration.")
         return LinterResult()

datacontract/lint/linters/quality_schema_linter.py

@@ -1,12 +1,11 @@
 import yaml
 
-from ..lint import Linter, LinterResult
-from datacontract.model.data_contract_specification import\
+from datacontract.model.data_contract_specification import \
     DataContractSpecification, Model
+from ..lint import Linter, LinterResult
 
 
 class QualityUsesSchemaLinter(Linter):
-
     @property
     def name(self) -> str:
         return "Quality check(s) use model"
@@ -15,23 +14,21 @@ class QualityUsesSchemaLinter(Linter):
     def id(self) -> str:
         return "quality-schema"
 
-    def lint_sodacl(self, check, models: dict[str, Model]) ->\
-            LinterResult:
+    def lint_sodacl(self, check, models: dict[str, Model]) -> LinterResult:
         result = LinterResult()
         for sodacl_check in check.keys():
-            table_name = sodacl_check[len("checks for "):]
+            table_name = sodacl_check[len("checks for ") :]
             if table_name not in models:
-                result = result.with_error(
-                    f"Quality check on unknown model '{table_name}'")
+                result = result.with_error(f"Quality check on unknown model '{table_name}'")
         return result
 
-    def lint_montecarlo(self, check, models: dict[str, Model]) ->\
-            LinterResult:
-        return LinterResult().with_warning(
-            "Linting montecarlo checks is not currently implemented")
+    def lint_montecarlo(self, check, models: dict[str, Model]) -> LinterResult:
+        return LinterResult().with_warning("Linting montecarlo checks is not currently implemented")
+
+    def lint_great_expectations(self, check, models: dict[str, Model]) -> LinterResult:
+        return LinterResult().with_warning("Linting great expectations checks is not currently implemented")
 
-    def lint_implementation(self, contract: DataContractSpecification) ->\
-            LinterResult:
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
         result = LinterResult()
         models = contract.models
         check = contract.quality
@@ -45,12 +42,11 @@ class QualityUsesSchemaLinter(Linter):
             check_specification = check.specification
         match check.type:
             case "SodaCL":
-                result = result.combine(
-                    self.lint_sodacl(check_specification, models))
+                result = result.combine(self.lint_sodacl(check_specification, models))
             case "montecarlo":
-                result = result.combine(
-                    self.lint_montecarlo(check_specification, models))
+                result = result.combine(self.lint_montecarlo(check_specification, models))
+            case "great-expectations":
+                result = result.combine(self.lint_great_expectations(check_specification, models))
             case _:
-                result = result.with_warning("Can't lint quality check "
-                                             f"with type '{check.type}'")
+                result = result.with_warning("Can't lint quality check " f"with type '{check.type}'")
         return result

datacontract/lint/linters/valid_constraints_linter.py

@@ -1,17 +1,16 @@
+from datacontract.model.data_contract_specification import DataContractSpecification, Field
 from ..lint import Linter, LinterResult
-from datacontract.model.data_contract_specification import\
-    DataContractSpecification, Field
 
 
 class ValidFieldConstraintsLinter(Linter):
     """Check validity of field constraints.
 
-      More precisely, check that only numeric constraints are specified on
-      fields of numeric type and string constraints on fields of string type.
-      Additionally, the linter checks that defined constraints make sense.
-      Minimum values should not be greater than maximum values, exclusive and
-      non-exclusive minimum and maximum should not be combined and string
-      pattern and format should not be combined.
+    More precisely, check that only numeric constraints are specified on
+    fields of numeric type and string constraints on fields of string type.
+    Additionally, the linter checks that defined constraints make sense.
+    Minimum values should not be greater than maximum values, exclusive and
+    non-exclusive minimum and maximum should not be combined and string
+    pattern and format should not be combined.
 
     """
 
@@ -20,52 +19,60 @@ class ValidFieldConstraintsLinter(Linter):
         "format": set(["string", "text", "varchar"]),
         "minLength": set(["string", "text", "varchar"]),
         "maxLength": set(["string", "text", "varchar"]),
-        "minimum": set(["int", "integer", "number", "decimal", "numeric",
-                        "long", "bigint", "float", "double"]),
-        "exclusiveMinimum": set(["int", "integer", "number", "decimal", "numeric",
-                                 "long", "bigint", "float", "double"]),
-        "maximum": set(["int", "integer", "number", "decimal", "numeric",
-                        "long", "bigint", "float", "double"]),
-        "exclusiveMaximum": set(["int", "integer", "number", "decimal", "numeric",
-                                 "long", "bigint", "float", "double"]),
+        "minimum": set(["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]),
+        "exclusiveMinimum": set(
+            ["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]
+        ),
+        "maximum": set(["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]),
+        "exclusiveMaximum": set(
+            ["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]
+        ),
     }
 
     def check_minimum_maximum(self, field: Field, field_name: str, model_name: str) -> LinterResult:
         (min, max, xmin, xmax) = (field.minimum, field.maximum, field.exclusiveMinimum, field.exclusiveMaximum)
-        match ("minimum" in field.model_fields_set, "maximum" in field.model_fields_set,
-               "exclusiveMinimum" in field.model_fields_set, "exclusiveMaximum" in field.model_fields_set):
+        match (
+            "minimum" in field.model_fields_set,
+            "maximum" in field.model_fields_set,
+            "exclusiveMinimum" in field.model_fields_set,
+            "exclusiveMaximum" in field.model_fields_set,
+        ):
             case (True, True, _, _) if min > max:
                 return LinterResult.erroneous(
-                    f"Minimum {min} is greater than maximum {max} on "
-                    f"field '{field_name}' in model '{model_name}'.")
+                    f"Minimum {min} is greater than maximum {max} on " f"field '{field_name}' in model '{model_name}'."
+                )
             case (_, _, True, True) if xmin >= xmax:
-                  return LinterResult.erroneous(
+                return LinterResult.erroneous(
                     f"Exclusive minimum {xmin} is greater than exclusive"
-                    f" maximum {xmax} on field '{field_name}' in model '{model_name}'.")
+                    f" maximum {xmax} on field '{field_name}' in model '{model_name}'."
+                )
             case (True, True, True, True):
                 return LinterResult.erroneous(
                     f"Both exclusive and non-exclusive minimum and maximum are "
-                    f"defined on field '{field_name}' in model '{model_name}'.")
+                    f"defined on field '{field_name}' in model '{model_name}'."
+                )
             case (True, _, True, _):
                 return LinterResult.erroneous(
                     f"Both exclusive and non-exclusive minimum are "
-                    f"defined on field '{field_name}' in model '{model_name}'.")
+                    f"defined on field '{field_name}' in model '{model_name}'."
+                )
             case (_, True, _, True):
                 return LinterResult.erroneous(
                     f"Both exclusive and non-exclusive maximum are "
-                    f"defined on field '{field_name}' in model '{model_name}'.")
+                    f"defined on field '{field_name}' in model '{model_name}'."
+                )
         return LinterResult()
 
     def check_string_constraints(self, field: Field, field_name: str, model_name: str) -> LinterResult:
         result = LinterResult()
         if field.minLength and field.maxLength and field.minLength > field.maxLength:
             result = result.with_error(
-                f"Minimum length is greater that maximum length on"
-                f" field '{field_name}' in model '{model_name}'.")
+                f"Minimum length is greater that maximum length on" f" field '{field_name}' in model '{model_name}'."
+            )
         if field.pattern and field.format:
             result = result.with_error(
-                f"Both a pattern and a format are defined for field"
-                f" '{field_name}' in model '{model_name}'.")
+                f"Both a pattern and a format are defined for field" f" '{field_name}' in model '{model_name}'."
+            )
         return result
 
     @property
@@ -76,19 +83,17 @@ class ValidFieldConstraintsLinter(Linter):
     def id(self):
         return "field-constraints"
 
-    def lint_implementation(
-            self,
-            contract: DataContractSpecification
-    ) -> LinterResult:
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
         result = LinterResult()
-        for (model_name, model) in contract.models.items():
-            for (field_name, field) in model.fields.items():
-                for (_property, allowed_types) in self.valid_types_for_constraint.items():
+        for model_name, model in contract.models.items():
+            for field_name, field in model.fields.items():
+                for _property, allowed_types in self.valid_types_for_constraint.items():
                     if _property in field.model_fields_set and field.type not in allowed_types:
                         result = result.with_error(
                             f"Forbidden constraint '{_property}' defined on field "
                             f"'{field_name}' in model '{model_name}'. Field type "
-                            f"is '{field.type}'.")
+                            f"is '{field.type}'."
+                        )
                 result = result.combine(self.check_minimum_maximum(field, field_name, model_name))
                 result = result.combine(self.check_string_constraints(field, field_name, model_name))
         return result

datacontract/lint/resolve.py

@@ -1,4 +1,5 @@
 import logging
+import os
 
 import fastjsonschema
 import yaml
@@ -8,7 +9,7 @@ from datacontract.lint.files import read_file
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
 from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Definition
+    DataContractSpecification, Definition, Quality
 from datacontract.model.exceptions import DataContractException
 
 
@@ -17,7 +18,7 @@ def resolve_data_contract(
     data_contract_str: str = None,
     data_contract: DataContractSpecification = None,
     schema_location: str = None,
-    inline_definitions: bool = False
+    inline_definitions: bool = False,
 ) -> DataContractSpecification:
     if data_contract_location is not None:
         return resolve_data_contract_from_location(data_contract_location, schema_location, inline_definitions)
@@ -30,20 +31,19 @@ def resolve_data_contract(
             type="lint",
             result="failed",
             name="Check that data contract YAML is valid",
-            reason=f"Data contract needs to be provided",
+            reason="Data contract needs to be provided",
             engine="datacontract",
         )
 
 
 def resolve_data_contract_from_location(
-    location, schema_location: str = None,
-    inline_definitions: bool = False
+    location, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = True
 ) -> DataContractSpecification:
     if location.startswith("http://") or location.startswith("https://"):
         data_contract_str = fetch_resource(location)
     else:
         data_contract_str = read_file(location)
-    return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions)
+    return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, include_quality)
 
 
 def inline_definitions_into_data_contract(spec: DataContractSpecification):
@@ -53,16 +53,15 @@ def inline_definitions_into_data_contract(spec: DataContractSpecification):
             if not field.ref and not field.ref_obj:
                 continue
 
-            definition = resolve_ref(field.ref, spec.definitions)
+            definition = resolve_definition_ref(field.ref, spec.definitions)
             field.ref_obj = definition
 
             for field_name in field.model_fields.keys():
-                if (field_name in definition.model_fields_set and
-                    field_name not in field.model_fields_set):
-                    setattr(field, field_name,
-                            getattr(definition, field_name))
+                if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
+                    setattr(field, field_name, getattr(definition, field_name))
 
-def resolve_ref(ref, definitions) -> Definition:
+
+def resolve_definition_ref(ref, definitions) -> Definition:
     if ref.startswith("http://") or ref.startswith("https://"):
         definition_str = fetch_resource(ref)
         definition_dict = to_yaml(definition_str)
@@ -81,9 +80,44 @@ def resolve_ref(ref, definitions) -> Definition:
         )
 
 
+def resolve_quality_ref(quality: Quality):
+    """
+    Return the content of a ref file path
+    @param quality data contract quality specification
+    """
+    if isinstance(quality.specification, dict):
+        specification = quality.specification
+        if quality.type == "great-expectations":
+            for model, model_quality in specification.items():
+                specification[model] = get_quality_ref_file(model_quality)
+        else:
+            if "$ref" in specification:
+                quality.specification = get_quality_ref_file(specification)
+
+
+def get_quality_ref_file(quality_spec: str | object) -> str | object:
+    """
+    Get the file associated with a quality reference
+    @param quality_spec quality specification
+    @returns: the content of the quality file
+    """
+    if isinstance(quality_spec, dict) and "$ref" in quality_spec:
+        ref = quality_spec["$ref"]
+        if not os.path.exists(ref):
+            raise DataContractException(
+                type="export",
+                result="failed",
+                name="Check that data contract quality is valid",
+                reason=f"Cannot resolve reference {ref}",
+                engine="datacontract",
+            )
+        with open(ref, "r") as file:
+            quality_spec = file.read()
+    return quality_spec
+
+
 def resolve_data_contract_from_str(
-    data_contract_str, schema_location: str = None,
-    inline_definitions: bool = False
+    data_contract_str, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = False
 ) -> DataContractSpecification:
     data_contract_yaml_dict = to_yaml(data_contract_str)
     validate(data_contract_yaml_dict, schema_location)
@@ -92,6 +126,8 @@ def resolve_data_contract_from_str(
 
     if inline_definitions:
         inline_definitions_into_data_contract(spec)
+    if spec.quality and include_quality:
+        resolve_quality_ref(spec.quality)
 
     return spec
 

datacontract/lint/schema.py

@@ -7,7 +7,6 @@ from datacontract.model.exceptions import DataContractException
 
 
 def fetch_schema(location: str = None):
-
     if location is None:
         location = "https://datacontract.com/datacontract.schema.json"
 
@@ -21,8 +20,8 @@ def fetch_schema(location: str = None):
                 name=f"Reading schema from {location}",
                 reason=f"The file '{location}' does not exist.",
                 engine="datacontract",
-                result="error"
+                result="error",
             )
-        with open(location, 'r') as file:
+        with open(location, "r") as file:
             file_content = file.read()
         return json.loads(file_content)

datacontract/lint/urls.py

@@ -20,22 +20,21 @@ def fetch_resource(url: str):
             name=f"Reading data contract from {url}",
             reason=f"Cannot read resource from URL {url}. Response status is {response.status_code}",
             engine="datacontract",
-            result="error"
+            result="error",
         )
 
 
 def _set_api_key(headers, url):
     if ".datamesh-manager.com/" not in url:
         return
-    datamesh_manager_api_key = os.getenv('DATAMESH_MANAGER_API_KEY')
+    datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
     if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
-        print(f"Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
+        print("Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
         raise DataContractException(
             type="lint",
             name=f"Reading data contract from {url}",
             reason="Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
             engine="datacontract",
-            result="error"
+            result="error",
         )
     headers["x-api-key"] = datamesh_manager_api_key
-

datacontract/model/breaking_change.py

@@ -1,7 +1,7 @@
+from enum import Enum
 from typing import List
 
 from pydantic import BaseModel
-from enum import Enum
 
 
 class Severity(Enum):
@@ -58,6 +58,7 @@ class BreakingChanges(BaseModel):
 
         return headline + content
 
+
 #
 # [
 #     {

datacontract/model/data_contract_specification.py

@@ -22,13 +22,13 @@ class Server(pyd.BaseModel):
     location: str = None
     account: str = None
     database: str = None
-    schema_: str = pyd.Field(default=None, alias='schema')
+    schema_: str = pyd.Field(default=None, alias="schema")
     host: str = None
     port: int = None
     catalog: str = None
     topic: str = None
-    http_path: str = None # Use ENV variable
-    token: str = None     # Use ENV variable
+    http_path: str = None  # Use ENV variable
+    token: str = None  # Use ENV variable
     dataProductId: str = None
     outputPortId: str = None
 
@@ -81,13 +81,14 @@ class Field(pyd.BaseModel):
     exclusiveMaximum: int = None
     enum: List[str] = []
     tags: List[str] = []
-    fields: Dict[str, 'Field'] = {}
-    items: 'Field' = None
+    fields: Dict[str, "Field"] = {}
+    items: "Field" = None
 
 
 class Model(pyd.BaseModel):
     description: str = None
     type: str = None
+    namespace: str = None
     fields: Dict[str, Field] = {}
 
 
@@ -126,8 +127,8 @@ class DataContractSpecification(pyd.BaseModel):
     @classmethod
     def from_file(cls, file):
         if not os.path.exists(file):
-            raise(f"The file '{file}' does not exist.")
-        with open(file, 'r') as file:
+            raise (f"The file '{file}' does not exist.")
+        with open(file, "r") as file:
             file_content = file.read()
         return DataContractSpecification.from_string(file_content)
 

datacontract/model/exceptions.py

@@ -11,7 +11,17 @@ class DataContractException(Exception):
         message (str): General message for the error.
     """
 
-    def __init__(self, type, name, reason, engine="datacontract", model=None, original_exception=None, result: str = "failed", message="Run operation failed"):
+    def __init__(
+        self,
+        type,
+        name,
+        reason,
+        engine="datacontract",
+        model=None,
+        original_exception=None,
+        result: str = "failed",
+        message="Run operation failed",
+    ):
         self.type = type
         self.name = name
         self.model = model
@@ -21,4 +31,5 @@ class DataContractException(Exception):
         self.original_exception = original_exception
         self.message = message
         super().__init__(
-            f"{self.message}: [{self.type}] {self.name} - {self.model} - {self.result} - {self.reason} - {self.engine}")
+            f"{self.message}: [{self.type}] {self.name} - {self.model} - {self.result} - {self.reason} - {self.engine}"
+        )

datacontract/model/run.py

@@ -15,6 +15,7 @@ class Check(BaseModel):
     model: Optional[str] = None
     field: Optional[str] = None
     details: Optional[str] = None
+    diagnostics: Optional[dict] = None
 
 
 class Log(BaseModel):
@@ -32,7 +33,7 @@ class Run(BaseModel):
     server: Optional[str] = None
     timestampStart: datetime
     timestampEnd: datetime
-    result: str = "unknown" # passed, warning, failed, error, unknown
+    result: str = "unknown"  # passed, warning, failed, error, unknown
     checks: List[Check]
     logs: List[Log]
 
@@ -69,7 +70,7 @@ class Run(BaseModel):
         self.logs.append(Log(level="ERROR", message=message, timestamp=datetime.now(timezone.utc)))
 
     def pretty(self):
-        return self.model_dump_json()
+        return self.model_dump_json(indent=2)
 
     @staticmethod
     def create_run():

datacontract/web.py

@@ -1,6 +1,6 @@
 from typing import Annotated, Union
 
-from fastapi import FastAPI, File, UploadFile
+from fastapi import FastAPI, File
 
 from datacontract.data_contract import DataContract
 
@@ -8,11 +8,7 @@ app = FastAPI()
 
 
 @app.post("/lint")
-def lint(file: Annotated[bytes, File()],
-         linters: Union[str, set[str]]="all"):
+def lint(file: Annotated[bytes, File()], linters: Union[str, set[str]] = "all"):
     data_contract = DataContract(data_contract_str=str(file, encoding="utf-8"))
     lint_result = data_contract.lint(enabled_linters=linters)
-    return {
-        "result": lint_result.result,
-        "checks": lint_result.checks
-    }
+    return {"result": lint_result.result, "checks": lint_result.checks}