datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/lint/lint.py

@@ -1,141 +0,0 @@
-import abc
-from dataclasses import dataclass, field
-from enum import Enum
-from typing import Sequence, Any, cast
-
-from datacontract.model.run import Check
-from ..model.data_contract_specification import DataContractSpecification
-
-"""This module contains linter definitions for linting a data contract.
-
-Lints are quality checks that can succeed, fail, or warn. They are
-distinct from checks such as "valid yaml" or "file not found", which
-will cause the processing of the data contract to stop. Lints can be
-ignored, and are high-level requirements on the format of a data
-contract."""
-
-
-class LintSeverity(Enum):
-    """The severity of a lint message. Generally, lint messages should be
-    emitted with a severity of ERROR. WARNING should be used when the linter
-    cannot determine a lint result, for example, when an unsupported model
-    type is used.
-    """
-
-    ERROR = 2
-    WARNING = 1
-
-
-@dataclass
-class LinterMessage:
-    """A single linter message with attached severity and optional "model" that
-    caused the message.
-
-    Attributes:
-       outcome: The outcome of the linting, either ERROR or WARNING. Linting outcomes with level WARNING are discarded for now.
-       message: A message describing the error or warning in more detail.
-       model: The model that caused the lint to fail. Is optional.
-
-    """
-
-    outcome: LintSeverity
-    message: str
-    model: Any = None
-
-    @classmethod
-    def error(cls, message: str, model=None):
-        return LinterMessage(LintSeverity.ERROR, message, model)
-
-    @classmethod
-    def warning(cls, message: str, model=None):
-        return LinterMessage(LintSeverity.WARNING, message, model)
-
-
-@dataclass
-class LinterResult:
-    """Result of linting a contract. Contains multiple LinterResults from
-       the same linter or lint phase.
-
-    Attributes:
-      linter: The linter that produced these results
-      results: A list of linting results. Multiple identical linting
-          results can be present in the list. An empty list means that
-          the linter ran without producing warnings or errors.
-    """
-
-    results: Sequence[LinterMessage] = field(default_factory=list)
-
-    @classmethod
-    def erroneous(cls, message, model=None):
-        return cls([LinterMessage.error(message, model)])
-
-    @classmethod
-    def cautious(cls, message, model=None):
-        return cls([LinterMessage.warning(message, model)])
-
-    def with_warning(self, message, model=None):
-        result = LinterMessage.warning(message, model)
-        return LinterResult(cast(list[LinterMessage], self.results) + [result])
-
-    def with_error(self, message, model=None):
-        result = LinterMessage.error(message, model)
-        return LinterResult(cast(list[LinterMessage], self.results) + [result])
-
-    def has_errors(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.ERROR, self.results))
-
-    def has_warnings(self) -> bool:
-        return any(map(lambda result: result.outcome == LintSeverity.WARNING, self.results))
-
-    def error_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results if result.outcome == LintSeverity.ERROR]
-
-    def warning_results(self) -> Sequence[LinterMessage]:
-        return [result for result in self.results if result.outcome == LintSeverity.WARNING]
-
-    def no_errors_or_warnings(self) -> bool:
-        return len(self.results) == 0
-
-    def combine(self, other: "LinterResult") -> "LinterResult":
-        return LinterResult(cast(list[Any], self.results) + cast(list[Any], other.results))
-
-
-class Linter(abc.ABC):
-    @property
-    @abc.abstractmethod
-    def name(self) -> str:
-        """Human-readable name of the linter."""
-        pass
-
-    @property
-    @abc.abstractmethod
-    def id(self) -> str:
-        """A linter ID for configuration (i.e. enabling and disabling)."""
-        pass
-
-    @abc.abstractmethod
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        pass
-
-    def lint(self, contract: DataContractSpecification) -> list[Check]:
-        """Call with a data contract to get a list of check results from the linter."""
-        result = self.lint_implementation(contract)
-        checks = []
-        if not result.error_results():
-            checks.append(Check(type="lint", name=f"Linter '{self.name}'", result="passed", engine="datacontract"))
-        else:
-            # All linter messages are treated as warnings. Severity is
-            # currently ignored, but could be used in filtering in the future
-            # Linter messages with level WARNING are currently ignored, but might
-            # be logged or printed in the future.
-            for lint_error in result.error_results():
-                checks.append(
-                    Check(
-                        type="lint",
-                        name=f"Linter '{self.name}'",
-                        result="warning",
-                        engine="datacontract",
-                        reason=lint_error.message,
-                    )
-                )
-        return checks

datacontract/lint/linters/description_linter.py

@@ -1,34 +0,0 @@
-from datacontract.model.data_contract_specification import DataContractSpecification
-from ..lint import Linter, LinterResult
-
-
-class DescriptionLinter(Linter):
-    """Check for a description on contracts, models, model fields, definitions and examples."""
-
-    @property
-    def name(self) -> str:
-        return "Objects have descriptions"
-
-    @property
-    def id(self) -> str:
-        return "description"
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        result = LinterResult()
-        if not contract.info or not contract.info.description:
-            result = result.with_error("Contract has empty description.")
-        for model_name, model in contract.models.items():
-            if not model.description:
-                result = result.with_error(f"Model '{model_name}' has empty description.")
-            for field_name, field in model.fields.items():
-                if not field.description:
-                    result = result.with_error(
-                        f"Field '{field_name}' in model '{model_name}'" f" has empty description."
-                    )
-        for definition_name, definition in contract.definitions.items():
-            if not definition.description:
-                result = result.with_error(f"Definition '{definition_name}' has empty description.")
-        for index, example in enumerate(contract.examples):
-            if not example.description:
-                result = result.with_error(f"Example {index + 1} has empty description.")
-        return result

datacontract/lint/linters/example_model_linter.py

@@ -1,91 +0,0 @@
-import csv
-import io
-import json
-
-import yaml
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Example
-from ..lint import Linter, LinterResult
-
-
-class ExampleModelLinter(Linter):
-    @property
-    def name(self) -> str:
-        return "Example(s) match model"
-
-    @property
-    def id(self) -> str:
-        return "example-model"
-
-    @staticmethod
-    def get_example_headers(example: Example) -> list[str]:
-        if isinstance(example.data, str):
-            match example.type:
-                case "csv":
-                    dialect = csv.Sniffer().sniff(example.data)
-                    data = io.StringIO(example.data)
-                    reader = csv.reader(data, dialect=dialect)
-                    return next(reader)
-                case "yaml":
-                    data = yaml.safe_load(example.data)
-                    return data.keys()
-                case "json":
-                    data = json.loads(example.data)
-                    return data.keys()
-                case _:
-                    # This is checked in lint_implementation, so shouldn't happen.
-                    raise NotImplementedError(f"Unknown type {example.type}")
-        else:
-            # Checked in lint_implementation, shouldn't happen.
-            raise NotImplementedError("Can't lint object examples.")
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        """Check whether the example(s) headers match the model.
-
-        This linter checks whether the example's fields match the model
-        fields, and whether all required fields of the model are present in
-        the example.
-        """
-        result = LinterResult()
-        examples = contract.examples
-        models = contract.models
-        examples_with_model = []
-        for index, example in enumerate(examples):
-            if example.model not in models:
-                result = result.with_error(f"Example {index + 1} has non-existent model '{example.model}'")
-            else:
-                examples_with_model.append((index, example, models.get(example.model)))
-        for index, example, model in examples_with_model:
-            if example.type == "custom":
-                result = result.with_warning(
-                    f"Example {index + 1} has type" ' "custom", cannot check model' " conformance"
-                )
-            elif not isinstance(example.data, str):
-                result = result.with_warning(
-                    f"Example {index + 1} is not a " "string example, can only lint string examples for now."
-                )
-            elif model.type == "object":
-                result = result.with_warning(
-                    f"Example {index + 1} uses a "
-                    f"model '{example.model}' with type 'object'. Linting is "
-                    "currently only supported for 'table' models"
-                )
-            else:
-                if example.type in ("csv", "yaml", "json"):
-                    headers = self.get_example_headers(example)
-                    for example_header in headers:
-                        if example_header not in model.fields:
-                            result = result.with_error(
-                                f"Example {index + 1} has field '{example_header}'"
-                                f" that's not contained in model '{example.model}'"
-                            )
-                    for field_name, field_value in model.fields.items():
-                        if field_name not in headers and field_value.required:
-                            result = result.with_error(
-                                f"Example {index + 1} is missing field '{field_name}'"
-                                f" required by model '{example.model}'"
-                            )
-                else:
-                    result = result.with_error(f"Example {index + 1} has unknown type" f"{example.type}")
-        return result

datacontract/lint/linters/field_pattern_linter.py

@@ -1,34 +0,0 @@
-import re
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
-from ..lint import Linter, LinterResult
-
-
-class FieldPatternLinter(Linter):
-    """Checks that all patterns defined for fields are correct Python regex
-    syntax.
-
-    """
-
-    @property
-    def name(self):
-        return "Field pattern is correct regex"
-
-    @property
-    def id(self) -> str:
-        return "field-pattern"
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        result = LinterResult()
-        for model_name, model in contract.models.items():
-            for field_name, field in model.fields.items():
-                if field.pattern:
-                    try:
-                        re.compile(field.pattern)
-                    except re.error as e:
-                        result = result.with_error(
-                            f"Failed to compile pattern regex '{field.pattern}' for "
-                            f"field '{field_name}' in model '{model_name}': {e.msg}"
-                        )
-        return result

datacontract/lint/linters/field_reference_linter.py

@@ -1,38 +0,0 @@
-from datacontract.model.data_contract_specification import DataContractSpecification
-from ..lint import Linter, LinterResult
-
-
-class FieldReferenceLinter(Linter):
-    """Checks that all references definitions in fields refer to existing
-    fields.
-
-    """
-
-    @property
-    def name(self):
-        return "Field references existing field"
-
-    @property
-    def id(self) -> str:
-        return "field-reference"
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        result = LinterResult()
-        for model_name, model in contract.models.items():
-            for field_name, field in model.fields.items():
-                if field.references:
-                    (ref_model, ref_field) = field.references.split(".", maxsplit=2)
-                    if ref_model not in contract.models:
-                        result = result.with_error(
-                            f"Field '{field_name}' in model '{model_name}'"
-                            f" references non-existing model '{ref_model}'."
-                        )
-                    else:
-                        ref_model_obj = contract.models[ref_model]
-                        if ref_field not in ref_model_obj.fields:
-                            result = result.with_error(
-                                f"Field '{field_name}' in model '{model_name}'"
-                                f" references non-existing field '{ref_field}'"
-                                f" in model '{ref_model}'."
-                            )
-        return result

datacontract/lint/linters/notice_period_linter.py

@@ -1,55 +0,0 @@
-import re
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification
-from ..lint import Linter, LinterResult
-
-
-class NoticePeriodLinter(Linter):
-    @property
-    def name(self) -> str:
-        return "noticePeriod in ISO8601 format"
-
-    @property
-    def id(self) -> str:
-        return "notice-period"
-
-    # Regex matching the "simple" ISO8601 duration format
-    simple = re.compile(
-        r"""P             # Introduces period
-        (:?[0-9\.,]+Y)?   # Number of years
-        (:?[0-9\.,]+M)?   # Number of months
-        (:?[0-9\.,]+W)?   # Number of weeks
-        (:?[0-9\.,]+D)?   # Number of days
-        (:?               # Time part (optional)
-          T               # Always starts with T
-          (:?[0-9\.,]+H)? # Number of hours
-          (:?[0-9\.,]+M)? # Number of minutes
-          (:?[0-9\.,]+S)? # Number of seconds
-        )?
-        """,
-        re.VERBOSE,
-    )
-    datetime_basic = re.compile(r"P\d{8}T\d{6}")
-    datetime_extended = re.compile(r"P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}")
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        """Check whether the notice period is specified using ISO8601 duration syntax."""
-        if not contract.terms:
-            return LinterResult.cautious("No terms defined.")
-        period = contract.terms.noticePeriod
-        if not period:
-            return LinterResult.cautious("No notice period defined.")
-        if not period.startswith("P"):
-            return LinterResult.erroneous(f"Notice period '{period}' is not a valid" "ISO8601 duration.")
-        if period == "P":
-            return LinterResult.erroneous(
-                "Notice period 'P' is not a valid" "ISO8601 duration, requires at least one" "duration to be specified."
-            )
-        if (
-            not self.simple.fullmatch(period)
-            and not self.datetime_basic.fullmatch(period)
-            and not self.datetime_extended.fullmatch(period)
-        ):
-            return LinterResult.erroneous(f"Notice period '{period}' is not a valid ISO8601 duration.")
-        return LinterResult()

datacontract/lint/linters/quality_schema_linter.py

@@ -1,52 +0,0 @@
-import yaml
-
-from datacontract.model.data_contract_specification import \
-    DataContractSpecification, Model
-from ..lint import Linter, LinterResult
-
-
-class QualityUsesSchemaLinter(Linter):
-    @property
-    def name(self) -> str:
-        return "Quality check(s) use model"
-
-    @property
-    def id(self) -> str:
-        return "quality-schema"
-
-    def lint_sodacl(self, check, models: dict[str, Model]) -> LinterResult:
-        result = LinterResult()
-        for sodacl_check in check.keys():
-            table_name = sodacl_check[len("checks for ") :]
-            if table_name not in models:
-                result = result.with_error(f"Quality check on unknown model '{table_name}'")
-        return result
-
-    def lint_montecarlo(self, check, models: dict[str, Model]) -> LinterResult:
-        return LinterResult().with_warning("Linting montecarlo checks is not currently implemented")
-
-    def lint_great_expectations(self, check, models: dict[str, Model]) -> LinterResult:
-        return LinterResult().with_warning("Linting great expectations checks is not currently implemented")
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        result = LinterResult()
-        models = contract.models
-        check = contract.quality
-        if not check:
-            return LinterResult()
-        if not check.specification:
-            return LinterResult.cautious("Quality check without specification.")
-        if isinstance(check.specification, str):
-            check_specification = yaml.safe_load(check.specification)
-        else:
-            check_specification = check.specification
-        match check.type:
-            case "SodaCL":
-                result = result.combine(self.lint_sodacl(check_specification, models))
-            case "montecarlo":
-                result = result.combine(self.lint_montecarlo(check_specification, models))
-            case "great-expectations":
-                result = result.combine(self.lint_great_expectations(check_specification, models))
-            case _:
-                result = result.with_warning("Can't lint quality check " f"with type '{check.type}'")
-        return result

datacontract/lint/linters/valid_constraints_linter.py

@@ -1,99 +0,0 @@
-from datacontract.model.data_contract_specification import DataContractSpecification, Field
-from ..lint import Linter, LinterResult
-
-
-class ValidFieldConstraintsLinter(Linter):
-    """Check validity of field constraints.
-
-    More precisely, check that only numeric constraints are specified on
-    fields of numeric type and string constraints on fields of string type.
-    Additionally, the linter checks that defined constraints make sense.
-    Minimum values should not be greater than maximum values, exclusive and
-    non-exclusive minimum and maximum should not be combined and string
-    pattern and format should not be combined.
-
-    """
-
-    valid_types_for_constraint = {
-        "pattern": set(["string", "text", "varchar"]),
-        "format": set(["string", "text", "varchar"]),
-        "minLength": set(["string", "text", "varchar"]),
-        "maxLength": set(["string", "text", "varchar"]),
-        "minimum": set(["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]),
-        "exclusiveMinimum": set(
-            ["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]
-        ),
-        "maximum": set(["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]),
-        "exclusiveMaximum": set(
-            ["int", "integer", "number", "decimal", "numeric", "long", "bigint", "float", "double"]
-        ),
-    }
-
-    def check_minimum_maximum(self, field: Field, field_name: str, model_name: str) -> LinterResult:
-        (min, max, xmin, xmax) = (field.minimum, field.maximum, field.exclusiveMinimum, field.exclusiveMaximum)
-        match (
-            "minimum" in field.model_fields_set,
-            "maximum" in field.model_fields_set,
-            "exclusiveMinimum" in field.model_fields_set,
-            "exclusiveMaximum" in field.model_fields_set,
-        ):
-            case (True, True, _, _) if min > max:
-                return LinterResult.erroneous(
-                    f"Minimum {min} is greater than maximum {max} on " f"field '{field_name}' in model '{model_name}'."
-                )
-            case (_, _, True, True) if xmin >= xmax:
-                return LinterResult.erroneous(
-                    f"Exclusive minimum {xmin} is greater than exclusive"
-                    f" maximum {xmax} on field '{field_name}' in model '{model_name}'."
-                )
-            case (True, True, True, True):
-                return LinterResult.erroneous(
-                    f"Both exclusive and non-exclusive minimum and maximum are "
-                    f"defined on field '{field_name}' in model '{model_name}'."
-                )
-            case (True, _, True, _):
-                return LinterResult.erroneous(
-                    f"Both exclusive and non-exclusive minimum are "
-                    f"defined on field '{field_name}' in model '{model_name}'."
-                )
-            case (_, True, _, True):
-                return LinterResult.erroneous(
-                    f"Both exclusive and non-exclusive maximum are "
-                    f"defined on field '{field_name}' in model '{model_name}'."
-                )
-        return LinterResult()
-
-    def check_string_constraints(self, field: Field, field_name: str, model_name: str) -> LinterResult:
-        result = LinterResult()
-        if field.minLength and field.maxLength and field.minLength > field.maxLength:
-            result = result.with_error(
-                f"Minimum length is greater that maximum length on" f" field '{field_name}' in model '{model_name}'."
-            )
-        if field.pattern and field.format:
-            result = result.with_error(
-                f"Both a pattern and a format are defined for field" f" '{field_name}' in model '{model_name}'."
-            )
-        return result
-
-    @property
-    def name(self):
-        return "Fields use valid constraints"
-
-    @property
-    def id(self):
-        return "field-constraints"
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        result = LinterResult()
-        for model_name, model in contract.models.items():
-            for field_name, field in model.fields.items():
-                for _property, allowed_types in self.valid_types_for_constraint.items():
-                    if _property in field.model_fields_set and field.type not in allowed_types:
-                        result = result.with_error(
-                            f"Forbidden constraint '{_property}' defined on field "
-                            f"'{field_name}' in model '{model_name}'. Field type "
-                            f"is '{field.type}'."
-                        )
-                result = result.combine(self.check_minimum_maximum(field, field_name, model_name))
-                result = result.combine(self.check_string_constraints(field, field_name, model_name))
-        return result