datacontract-cli 0.9.6.post2__py3-none-any.whl → 0.9.8__py3-none-any.whl

datacontract/lint/linters/example_model_linter.py

@@ -1,67 +1,91 @@
 import csv
-import yaml
-import json
 import io
+import json
+
+import yaml
 
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Example
 from ..lint import Linter, LinterResult
-from datacontract.model.data_contract_specification import DataContractSpecification, Example
 
 
 class ExampleModelLinter(Linter):
+    @property
     def name(self) -> str:
         return "Example(s) match model"
 
+    @property
+    def id(self) -> str:
+        return "example-model"
+
     @staticmethod
     def get_example_headers(example: Example) -> list[str]:
-        match example.type:
-            case "csv":
-                dialect = csv.Sniffer().sniff(example.data)
-                data = io.StringIO(example.data)
-                reader = csv.reader(data, dialect=dialect)
-                return next(reader)
-            case "yaml":
-                data = yaml.safe_load(example.data)
-                return data.keys()
-            case "json":
-                data = json.loads(example.data)
-                return data.keys()
+        if isinstance(example.data, str):
+            match example.type:
+                case "csv":
+                    dialect = csv.Sniffer().sniff(example.data)
+                    data = io.StringIO(example.data)
+                    reader = csv.reader(data, dialect=dialect)
+                    return next(reader)
+                case "yaml":
+                    data = yaml.safe_load(example.data)
+                    return data.keys()
+                case "json":
+                    data = json.loads(example.data)
+                    return data.keys()
+                case _:
+                    # This is checked in lint_implementation, so shouldn't happen.
+                    raise NotImplementedError(f"Unknown type {example.type}")
+        else:
+            # Checked in lint_implementation, shouldn't happen.
+            raise NotImplementedError("Can't lint object examples.")
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        """Check whether the example(s) headers match the model.
 
-    def lint_implementation(
-        self,
-        data_contract_yaml: DataContractSpecification
-    ) -> LinterResult:
-        """Check whether the example(s) match the model."""
+        This linter checks whether the example's fields match the model
+        fields, and whether all required fields of the model are present in
+        the example.
+        """
         result = LinterResult()
-        examples = data_contract_yaml.examples
-        models = data_contract_yaml.models
+        examples = contract.examples
+        models = contract.models
         examples_with_model = []
-        for (index, example) in enumerate(examples):
+        for index, example in enumerate(examples):
             if example.model not in models:
-                result = result.with_error(
-                    f"Example {index + 1} has non-existent model '{example.model}'")
+                result = result.with_error(f"Example {index + 1} has non-existent model '{example.model}'")
             else:
-                examples_with_model.append(
-                    (index, example, models.get(example.model)))
-        for (index, example, model) in examples_with_model:
+                examples_with_model.append((index, example, models.get(example.model)))
+        for index, example, model in examples_with_model:
             if example.type == "custom":
-                result = result.with_warning(f"Example {index + 1} has type"
-                                             " \"custom\", cannot check model"
-                                             " conformance")
+                result = result.with_warning(
+                    f"Example {index + 1} has type" ' "custom", cannot check model' " conformance"
+                )
+            elif not isinstance(example.data, str):
+                result = result.with_warning(
+                    f"Example {index + 1} is not a " "string example, can only lint string examples for now."
+                )
             elif model.type == "object":
                 result = result.with_warning(
                     f"Example {index + 1} uses a "
                     f"model '{example.model}' with type 'object'. Linting is "
-                    "currently only supported for 'table' models")
+                    "currently only supported for 'table' models"
+                )
             else:
-                headers = self.get_example_headers(example)
-                for example_header in headers:
-                    if example_header not in model.fields:
-                        result = result.with_error(
-                            f"Example {index + 1} has field '{example_header}'"
-                            f" that's not contained in model '{example.model}'")
-                for (field_name, field_value) in model.fields.items():
-                    if field_name not in headers and field_value.required:
-                        result = result.with_error(
-                            f"Example {index + 1} is missing field '{field_name}'"
-                            f" required by model '{example.model}'")
+                if example.type in ("csv", "yaml", "json"):
+                    headers = self.get_example_headers(example)
+                    for example_header in headers:
+                        if example_header not in model.fields:
+                            result = result.with_error(
+                                f"Example {index + 1} has field '{example_header}'"
+                                f" that's not contained in model '{example.model}'"
+                            )
+                    for field_name, field_value in model.fields.items():
+                        if field_name not in headers and field_value.required:
+                            result = result.with_error(
+                                f"Example {index + 1} is missing field '{field_name}'"
+                                f" required by model '{example.model}'"
+                            )
+                else:
+                    result = result.with_error(f"Example {index + 1} has unknown type" f"{example.type}")
         return result

datacontract/lint/linters/field_pattern_linter.py

@@ -0,0 +1,34 @@
+import re
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification
+from ..lint import Linter, LinterResult
+
+
+class FieldPatternLinter(Linter):
+    """Checks that all patterns defined for fields are correct Python regex
+    syntax.
+
+    """
+
+    @property
+    def name(self):
+        return "Field pattern is correct regex"
+
+    @property
+    def id(self) -> str:
+        return "field-pattern"
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        result = LinterResult()
+        for model_name, model in contract.models.items():
+            for field_name, field in model.fields.items():
+                if field.pattern:
+                    try:
+                        re.compile(field.pattern)
+                    except re.error as e:
+                        result = result.with_error(
+                            f"Failed to compile pattern regex '{field.pattern}' for "
+                            f"field '{field_name}' in model '{model_name}': {e.msg}"
+                        )
+        return result

datacontract/lint/linters/field_reference_linter.py

@@ -0,0 +1,38 @@
+from datacontract.model.data_contract_specification import DataContractSpecification
+from ..lint import Linter, LinterResult
+
+
+class FieldReferenceLinter(Linter):
+    """Checks that all references definitions in fields refer to existing
+    fields.
+
+    """
+
+    @property
+    def name(self):
+        return "Field references existing field"
+
+    @property
+    def id(self) -> str:
+        return "field-reference"
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        result = LinterResult()
+        for model_name, model in contract.models.items():
+            for field_name, field in model.fields.items():
+                if field.references:
+                    (ref_model, ref_field) = field.references.split(".", maxsplit=2)
+                    if ref_model not in contract.models:
+                        result = result.with_error(
+                            f"Field '{field_name}' in model '{model_name}'"
+                            f" references non-existing model '{ref_model}'."
+                        )
+                    else:
+                        ref_model_obj = contract.models[ref_model]
+                        if ref_field not in ref_model_obj.fields:
+                            result = result.with_error(
+                                f"Field '{field_name}' in model '{model_name}'"
+                                f" references non-existing field '{ref_field}'"
+                                f" in model '{ref_model}'."
+                            )
+        return result

datacontract/lint/linters/notice_period_linter.py

@@ -0,0 +1,55 @@
+import re
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification
+from ..lint import Linter, LinterResult
+
+
+class NoticePeriodLinter(Linter):
+    @property
+    def name(self) -> str:
+        return "noticePeriod in ISO8601 format"
+
+    @property
+    def id(self) -> str:
+        return "notice-period"
+
+    # Regex matching the "simple" ISO8601 duration format
+    simple = re.compile(
+        r"""P             # Introduces period
+        (:?[0-9\.,]+Y)?   # Number of years
+        (:?[0-9\.,]+M)?   # Number of months
+        (:?[0-9\.,]+W)?   # Number of weeks
+        (:?[0-9\.,]+D)?   # Number of days
+        (:?               # Time part (optional)
+          T               # Always starts with T
+          (:?[0-9\.,]+H)? # Number of hours
+          (:?[0-9\.,]+M)? # Number of minutes
+          (:?[0-9\.,]+S)? # Number of seconds
+        )?
+        """,
+        re.VERBOSE,
+    )
+    datetime_basic = re.compile(r"P\d{8}T\d{6}")
+    datetime_extended = re.compile(r"P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}")
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        """Check whether the notice period is specified using ISO8601 duration syntax."""
+        if not contract.terms:
+            return LinterResult.cautious("No terms defined.")
+        period = contract.terms.noticePeriod
+        if not period:
+            return LinterResult.cautious("No notice period defined.")
+        if not period.startswith("P"):
+            return LinterResult.erroneous(f"Notice period '{period}' is not a valid" "ISO8601 duration.")
+        if period == "P":
+            return LinterResult.erroneous(
+                "Notice period 'P' is not a valid" "ISO8601 duration, requires at least one" "duration to be specified."
+            )
+        if (
+            not self.simple.fullmatch(period)
+            and not self.datetime_basic.fullmatch(period)
+            and not self.datetime_extended.fullmatch(period)
+        ):
+            return LinterResult.erroneous(f"Notice period '{period}' is not a valid ISO8601 duration.")
+        return LinterResult()

datacontract/lint/linters/primary_field_linter.py

@@ -0,0 +1,28 @@
+from datacontract.model.data_contract_specification import DataContractSpecification
+from ..lint import Linter, LinterResult
+
+
+class PrimaryFieldUniqueRequired(Linter):
+    """Checks that all fields defined as primary are also defined as unique and required."""
+
+    @property
+    def name(self) -> str:
+        return "Model primary fields unique and required"
+
+    @property
+    def id(self) -> str:
+        return "notice-period"
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        if not contract.models:
+            return LinterResult.cautious("No models defined on contract.")
+        result = LinterResult()
+        for model_name, model in contract.models.items():
+            for field_name, field in model.fields.items():
+                if field.primary and not field.required and not field.unique:
+                    result = result.with_error(
+                        f"Field '{field_name}' in model '{model_name}'"
+                        " is marked as primary, but not as unique"
+                        " and required."
+                    )
+        return result

datacontract/lint/linters/quality_schema_linter.py

@@ -0,0 +1,52 @@
+import yaml
+
+from datacontract.model.data_contract_specification import \
+    DataContractSpecification, Model
+from ..lint import Linter, LinterResult
+
+
+class QualityUsesSchemaLinter(Linter):
+    @property
+    def name(self) -> str:
+        return "Quality check(s) use model"
+
+    @property
+    def id(self) -> str:
+        return "quality-schema"
+
+    def lint_sodacl(self, check, models: dict[str, Model]) -> LinterResult:
+        result = LinterResult()
+        for sodacl_check in check.keys():
+            table_name = sodacl_check[len("checks for ") :]
+            if table_name not in models:
+                result = result.with_error(f"Quality check on unknown model '{table_name}'")
+        return result
+
+    def lint_montecarlo(self, check, models: dict[str, Model]) -> LinterResult:
+        return LinterResult().with_warning("Linting montecarlo checks is not currently implemented")
+
+    def lint_great_expectations(self, check, models: dict[str, Model]) -> LinterResult:
+        return LinterResult().with_warning("Linting great expectations checks is not currently implemented")
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        result = LinterResult()
+        models = contract.models
+        check = contract.quality
+        if not check:
+            return LinterResult()
+        if not check.specification:
+            return LinterResult.cautious("Quality check without specification.")
+        if isinstance(check.specification, str):
+            check_specification = yaml.safe_load(check.specification)
+        else:
+            check_specification = check.specification
+        match check.type:
+            case "SodaCL":
+                result = result.combine(self.lint_sodacl(check_specification, models))
+            case "montecarlo":
+                result = result.combine(self.lint_montecarlo(check_specification, models))
+            case "great-expectations":
+                result = result.combine(self.lint_great_expectations(check_specification, models))
+            case _:
+                result = result.with_warning("Can't lint quality check " f"with type '{check.type}'")
+        return result

datacontract/lint/linters/valid_constraints_linter.py

@@ -0,0 +1,99 @@
+from datacontract.model.data_contract_specification import DataContractSpecification, Field
+from ..lint import Linter, LinterResult
+
+
+class ValidFieldConstraintsLinter(Linter):
+    """Check validity of field constraints.
+
+    More precisely, check that only numeric constraints are specified on
+    fields of numeric type and string constraints on fields of string type.
+    Additionally, the linter checks that defined constraints make sense.
+    Minimum values should not be greater than maximum values, exclusive and
+    non-exclusive minimum and maximum should not be combined and string
+    pattern and format should not be combined.
+
+    """
+
+    valid_types_for_constraint = {
+        "pattern": set(["string", "text", "varchar"]),
+        "format": set(["string", "text", "varchar"]),
+        "minLength": set(["string", "text", "varchar"]),
+        "maxLength": set(["string", "text", "varchar"]),
+        "minimum": set(["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]),
+        "exclusiveMinimum": set(
+            ["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]
+        ),
+        "maximum": set(["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]),
+        "exclusiveMaximum": set(
+            ["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]
+        ),
+    }
+
+    def check_minimum_maximum(self, field: Field, field_name: str, model_name: str) -> LinterResult:
+        (min, max, xmin, xmax) = (field.minimum, field.maximum, field.exclusiveMinimum, field.exclusiveMaximum)
+        match (
+            "minimum" in field.model_fields_set,
+            "maximum" in field.model_fields_set,
+            "exclusiveMinimum" in field.model_fields_set,
+            "exclusiveMaximum" in field.model_fields_set,
+        ):
+            case (True, True, _, _) if min > max:
+                return LinterResult.erroneous(
+                    f"Minimum {min} is greater than maximum {max} on " f"field '{field_name}' in model '{model_name}'."
+                )
+            case (_, _, True, True) if xmin >= xmax:
+                return LinterResult.erroneous(
+                    f"Exclusive minimum {xmin} is greater than exclusive"
+                    f" maximum {xmax} on field '{field_name}' in model '{model_name}'."
+                )
+            case (True, True, True, True):
+                return LinterResult.erroneous(
+                    f"Both exclusive and non-exclusive minimum and maximum are "
+                    f"defined on field '{field_name}' in model '{model_name}'."
+                )
+            case (True, _, True, _):
+                return LinterResult.erroneous(
+                    f"Both exclusive and non-exclusive minimum are "
+                    f"defined on field '{field_name}' in model '{model_name}'."
+                )
+            case (_, True, _, True):
+                return LinterResult.erroneous(
+                    f"Both exclusive and non-exclusive maximum are "
+                    f"defined on field '{field_name}' in model '{model_name}'."
+                )
+        return LinterResult()
+
+    def check_string_constraints(self, field: Field, field_name: str, model_name: str) -> LinterResult:
+        result = LinterResult()
+        if field.minLength and field.maxLength and field.minLength > field.maxLength:
+            result = result.with_error(
+                f"Minimum length is greater that maximum length on" f" field '{field_name}' in model '{model_name}'."
+            )
+        if field.pattern and field.format:
+            result = result.with_error(
+                f"Both a pattern and a format are defined for field" f" '{field_name}' in model '{model_name}'."
+            )
+        return result
+
+    @property
+    def name(self):
+        return "Fields use valid constraints"
+
+    @property
+    def id(self):
+        return "field-constraints"
+
+    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
+        result = LinterResult()
+        for model_name, model in contract.models.items():
+            for field_name, field in model.fields.items():
+                for _property, allowed_types in self.valid_types_for_constraint.items():
+                    if _property in field.model_fields_set and field.type not in allowed_types:
+                        result = result.with_error(
+                            f"Forbidden constraint '{_property}' defined on field "
+                            f"'{field_name}' in model '{model_name}'. Field type "
+                            f"is '{field.type}'."
+                        )
+                result = result.combine(self.check_minimum_maximum(field, field_name, model_name))
+                result = result.combine(self.check_string_constraints(field, field_name, model_name))
+        return result

datacontract/lint/resolve.py

@@ -8,7 +8,7 @@ from datacontract.lint.files import read_file
 from datacontract.lint.schema import fetch_schema
 from datacontract.lint.urls import fetch_resource
 from datacontract.model.data_contract_specification import \
-    DataContractSpecification
+    DataContractSpecification, Definition
 from datacontract.model.exceptions import DataContractException
 
 
@@ -17,11 +17,12 @@ def resolve_data_contract(
     data_contract_str: str = None,
     data_contract: DataContractSpecification = None,
     schema_location: str = None,
+    inline_definitions: bool = False,
 ) -> DataContractSpecification:
     if data_contract_location is not None:
-        return resolve_data_contract_from_location(data_contract_location, schema_location)
+        return resolve_data_contract_from_location(data_contract_location, schema_location, inline_definitions)
     elif data_contract_str is not None:
-        return resolve_data_contract_from_str(data_contract_str, schema_location)
+        return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions)
     elif data_contract is not None:
         return data_contract
     else:
@@ -29,23 +30,67 @@ def resolve_data_contract(
             type="lint",
             result="failed",
             name="Check that data contract YAML is valid",
-            reason=f"Data contract needs to be provided",
+            reason="Data contract needs to be provided",
             engine="datacontract",
         )
 
 
-def resolve_data_contract_from_location(location, schema_location: str = None) -> DataContractSpecification:
+def resolve_data_contract_from_location(
+    location, schema_location: str = None, inline_definitions: bool = False
+) -> DataContractSpecification:
     if location.startswith("http://") or location.startswith("https://"):
         data_contract_str = fetch_resource(location)
     else:
         data_contract_str = read_file(location)
-    return resolve_data_contract_from_str(data_contract_str, schema_location)
+    return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions)
+
+
+def inline_definitions_into_data_contract(spec: DataContractSpecification):
+    for model in spec.models.values():
+        for field in model.fields.values():
+            # If ref_obj is not empty, we've already inlined definitions.
+            if not field.ref and not field.ref_obj:
+                continue
+
+            definition = resolve_ref(field.ref, spec.definitions)
+            field.ref_obj = definition
+
+            for field_name in field.model_fields.keys():
+                if field_name in definition.model_fields_set and field_name not in field.model_fields_set:
+                    setattr(field, field_name, getattr(definition, field_name))
 
 
-def resolve_data_contract_from_str(data_contract_str, schema_location: str = None):
+def resolve_ref(ref, definitions) -> Definition:
+    if ref.startswith("http://") or ref.startswith("https://"):
+        definition_str = fetch_resource(ref)
+        definition_dict = to_yaml(definition_str)
+        return Definition(**definition_dict)
+
+    elif ref.startswith("#/definitions/"):
+        definition_name = ref.split("#/definitions/")[1]
+        return definitions[definition_name]
+    else:
+        raise DataContractException(
+            type="lint",
+            result="failed",
+            name="Check that data contract YAML is valid",
+            reason=f"Cannot resolve reference {ref}",
+            engine="datacontract",
+        )
+
+
+def resolve_data_contract_from_str(
+    data_contract_str, schema_location: str = None, inline_definitions: bool = False
+) -> DataContractSpecification:
     data_contract_yaml_dict = to_yaml(data_contract_str)
     validate(data_contract_yaml_dict, schema_location)
-    return DataContractSpecification(**data_contract_yaml_dict)
+
+    spec = DataContractSpecification(**data_contract_yaml_dict)
+
+    if inline_definitions:
+        inline_definitions_into_data_contract(spec)
+
+    return spec
 
 
 def to_yaml(data_contract_str):

datacontract/lint/schema.py

@@ -7,7 +7,6 @@ from datacontract.model.exceptions import DataContractException
 
 
 def fetch_schema(location: str = None):
-
     if location is None:
         location = "https://datacontract.com/datacontract.schema.json"
 
@@ -21,8 +20,8 @@ def fetch_schema(location: str = None):
                 name=f"Reading schema from {location}",
                 reason=f"The file '{location}' does not exist.",
                 engine="datacontract",
-                result="error"
+                result="error",
             )
-        with open(location, 'r') as file:
+        with open(location, "r") as file:
             file_content = file.read()
         return json.loads(file_content)

datacontract/lint/urls.py

@@ -20,22 +20,21 @@ def fetch_resource(url: str):
             name=f"Reading data contract from {url}",
             reason=f"Cannot read resource from URL {url}. Response status is {response.status_code}",
             engine="datacontract",
-            result="error"
+            result="error",
         )
 
 
 def _set_api_key(headers, url):
     if ".datamesh-manager.com/" not in url:
         return
-    datamesh_manager_api_key = os.getenv('DATAMESH_MANAGER_API_KEY')
+    datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
     if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
-        print(f"Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
+        print("Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
         raise DataContractException(
             type="lint",
             name=f"Reading data contract from {url}",
             reason="Error: Data Mesh Manager API Key is not set. Set env variable DATAMESH_MANAGER_API_KEY.",
             engine="datacontract",
-            result="error"
+            result="error",
         )
     headers["x-api-key"] = datamesh_manager_api_key
-

datacontract/model/breaking_change.py

@@ -1,8 +1,18 @@
+from enum import Enum
 from typing import List
 
 from pydantic import BaseModel
 
 
+class Severity(Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+    def __str__(self) -> str:
+        return self.value
+
+
 class Location(BaseModel):
     path: str
     composition: List[str]
@@ -10,7 +20,7 @@ class Location(BaseModel):
 
 class BreakingChange(BaseModel):
     description: str
-    severity: str
+    severity: Severity
     check_name: str
     location: Location
 
@@ -24,19 +34,31 @@ class BreakingChanges(BaseModel):
     breaking_changes: List[BreakingChange]
 
     def passed_checks(self) -> bool:
-        errors = len(list(filter(lambda x: x.severity == "error", self.breaking_changes)))
+        errors = len(list(filter(lambda x: x.severity == Severity.ERROR, self.breaking_changes)))
         return errors == 0
 
-    def __str__(self) -> str:
+    def breaking_str(self) -> str:
         changes_amount = len(self.breaking_changes)
-        errors = len(list(filter(lambda x: x.severity == "error", self.breaking_changes)))
-        warnings = len(list(filter(lambda x: x.severity == "warning", self.breaking_changes)))
+        errors = len(list(filter(lambda x: x.severity == Severity.ERROR, self.breaking_changes)))
+        warnings = len(list(filter(lambda x: x.severity == Severity.WARNING, self.breaking_changes)))
 
         headline = f"{changes_amount} breaking changes: {errors} error, {warnings} warning\n"
         content = str.join("\n\n", map(lambda x: str(x), self.breaking_changes))
 
         return headline + content
 
+    def changelog_str(self) -> str:
+        changes_amount = len(self.breaking_changes)
+        errors = len(list(filter(lambda x: x.severity == Severity.ERROR, self.breaking_changes)))
+        warnings = len(list(filter(lambda x: x.severity == Severity.WARNING, self.breaking_changes)))
+        infos = len(list(filter(lambda x: x.severity == Severity.INFO, self.breaking_changes)))
+
+        headline = f"{changes_amount} changes: {errors} error, {warnings} warning, {infos} info\n"
+        content = str.join("\n\n", map(lambda x: str(x), self.breaking_changes))
+
+        return headline + content
+
+
 #
 # [
 #     {