cognite-neat 0.88.1__py3-none-any.whl → 0.88.3__py3-none-any.whl

cognite/neat/issues/_base.py

@@ -0,0 +1,415 @@
+import sys
+import warnings
+from abc import ABC
+from collections import UserList
+from collections.abc import Collection, Hashable, Iterable, Sequence
+from dataclasses import dataclass, fields
+from functools import total_ordering
+from pathlib import Path
+from types import UnionType
+from typing import Any, ClassVar, Literal, TypeAlias, TypeVar, get_args, get_origin
+from warnings import WarningMessage
+
+import pandas as pd
+from cognite.client.data_classes.data_modeling import ContainerId, ViewId
+from pydantic_core import ErrorDetails
+
+from cognite.neat.utils.spreadsheet import SpreadsheetRead
+from cognite.neat.utils.text import humanize_collection, to_camel, to_snake
+
+if sys.version_info < (3, 11):
+    from exceptiongroup import ExceptionGroup
+    from typing_extensions import Self
+else:
+    from typing import Self
+
+
+__all__ = [
+    "NeatIssue",
+    "NeatError",
+    "NeatWarning",
+    "DefaultWarning",
+    "NeatIssueList",
+    "MultiValueError",
+]
+
+T_Identifier = TypeVar("T_Identifier", bound=Hashable)
+
+T_ReferenceIdentifier = TypeVar("T_ReferenceIdentifier", bound=Hashable)
+
+ResourceType: TypeAlias = (
+    Literal[
+        "view",
+        "container",
+        "view property",
+        "container property",
+        "space",
+        "class",
+        "asset",
+        "relationship",
+        "data model",
+        "edge",
+        "node",
+        "unknown",
+    ]
+    # String to handle all unknown types in different importers.
+    | str
+)
+
+
+@total_ordering
+@dataclass(frozen=True)
+class NeatIssue:
+    """This is the base class for all exceptions and warnings (issues) used in Neat."""
+
+    extra: ClassVar[str | None] = None
+    fix: ClassVar[str | None] = None
+
+    def as_message(self) -> str:
+        """Return a human-readable message for the issue."""
+        template = self.__doc__
+        if not template:
+            return "Missing"
+        variables, has_all_optional = self._get_variables()
+
+        msg = template.format(**variables)
+        if self.extra and has_all_optional:
+            msg += "\n" + self.extra.format(**variables)
+        if self.fix:
+            msg += f"\nFix: {self.fix.format(**variables)}"
+        name = type(self).__name__
+        return f"{name}: {msg}"
+
+    def _get_variables(self) -> tuple[dict[str, str], bool]:
+        variables: dict[str, str] = {}
+        has_all_optional = True
+        for name, var_ in vars(self).items():
+            if var_ is None:
+                has_all_optional = False
+            elif isinstance(var_, str):
+                variables[name] = var_
+            elif isinstance(var_, Path):
+                variables[name] = var_.as_posix()
+            elif isinstance(var_, Collection):
+                variables[name] = humanize_collection(var_)
+            else:
+                variables[name] = repr(var_)
+        return variables, has_all_optional
+
+    def dump(self) -> dict[str, Any]:
+        """Return a dictionary representation of the issue."""
+        variables = vars(self)
+        output = {to_camel(key): self._dump_value(value) for key, value in variables.items() if value is not None}
+        output["NeatIssue"] = type(self).__name__
+        return output
+
+    @classmethod
+    def _dump_value(cls, value: Any) -> list | int | bool | float | str | dict:
+        if isinstance(value, str | int | bool | float):
+            return value
+        elif isinstance(value, frozenset):
+            return [cls._dump_value(item) for item in value]
+        elif isinstance(value, Path):
+            return value.as_posix()
+        elif isinstance(value, tuple):
+            return [cls._dump_value(item) for item in value]
+        elif isinstance(value, ViewId | ContainerId):
+            return value.dump(camel_case=True, include_type=True)
+        raise ValueError(f"Unsupported type: {type(value)}")
+
+    @classmethod
+    def load(cls, data: dict[str, Any]) -> "NeatIssue":
+        """Create an instance of the issue from a dictionary."""
+        from cognite.neat.issues.errors import _NEAT_ERRORS_BY_NAME, NeatValueError
+        from cognite.neat.issues.warnings import _NEAT_WARNINGS_BY_NAME
+
+        if "NeatIssue" not in data:
+            raise NeatValueError("The data does not contain a NeatIssue key.")
+        issue_type = data.pop("NeatIssue")
+        args = {to_snake(key): value for key, value in data.items()}
+        if issue_type in _NEAT_ERRORS_BY_NAME:
+            return cls._load_values(_NEAT_ERRORS_BY_NAME[issue_type], args)
+        elif issue_type in _NEAT_WARNINGS_BY_NAME:
+            return cls._load_values(_NEAT_WARNINGS_BY_NAME[issue_type], args)
+        else:
+            raise NeatValueError(f"Unknown issue type: {issue_type}")
+
+    @classmethod
+    def _load_values(cls, neat_issue_cls: "type[NeatIssue]", data: dict[str, Any]) -> "NeatIssue":
+        args: dict[str, Any] = {}
+        for f in fields(neat_issue_cls):
+            if f.name not in data:
+                continue
+            value = data[f.name]
+            args[f.name] = cls._load_value(f.type, value)
+        return neat_issue_cls(**args)
+
+    @classmethod
+    def _load_value(cls, type_: type, value: Any) -> Any:
+        if isinstance(type_, UnionType) or get_origin(type_) is UnionType:
+            args = get_args(type_)
+            return cls._load_value(args[0], value)
+        elif type_ is frozenset or get_origin(type_) is frozenset:
+            subtype = get_args(type_)[0]
+            return frozenset(cls._load_value(subtype, item) for item in value)
+        elif type_ is Path:
+            return Path(value)
+        elif type_ is tuple or get_origin(type_) is tuple:
+            subtype = get_args(type_)[0]
+            return tuple(cls._load_value(subtype, item) for item in value)
+        elif type_ is ViewId:
+            return ViewId.load(value)
+        elif type_ is ContainerId:
+            return ContainerId.load(value)
+        return value
+
+    def __lt__(self, other: "NeatIssue") -> bool:
+        if not isinstance(other, NeatIssue):
+            return NotImplemented
+        return (type(self).__name__, self.as_message()) < (type(other).__name__, other.as_message())
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, NeatIssue):
+            return NotImplemented
+        return (type(self).__name__, self.as_message()) == (type(other).__name__, other.as_message())
+
+
+@dataclass(frozen=True)
+class NeatError(NeatIssue, Exception):
+    """This is the base class for all exceptions (errors) used in Neat."""
+
+    @classmethod
+    def from_pydantic_errors(cls, errors: list[ErrorDetails], **kwargs) -> "list[NeatError]":
+        """Convert a list of pydantic errors to a list of Error instances.
+
+        This is intended to be overridden in subclasses to handle specific error types.
+        """
+        all_errors: list[NeatError] = []
+        read_info_by_sheet = kwargs.get("read_info_by_sheet")
+
+        for error in errors:
+            ctx = error.get("ctx")
+            if isinstance(ctx, dict) and isinstance(multi_error := ctx.get("error"), MultiValueError):
+                if read_info_by_sheet:
+                    for caught_error in multi_error.errors:
+                        cls._adjust_row_numbers(caught_error, read_info_by_sheet)  # type: ignore[arg-type]
+                all_errors.extend(multi_error.errors)  # type: ignore[arg-type]
+            elif isinstance(ctx, dict) and isinstance(single_error := ctx.get("error"), NeatError):
+                if read_info_by_sheet:
+                    cls._adjust_row_numbers(single_error, read_info_by_sheet)
+                all_errors.append(single_error)
+            elif len(error["loc"]) >= 4 and read_info_by_sheet:
+                all_errors.append(RowError.from_pydantic_error(error, read_info_by_sheet))
+            else:
+                all_errors.append(DefaultPydanticError.from_pydantic_error(error))
+        return all_errors
+
+    @staticmethod
+    def _adjust_row_numbers(caught_error: "NeatError", read_info_by_sheet: dict[str, SpreadsheetRead]) -> None:
+        from cognite.neat.issues.errors._properties import PropertyDefinitionDuplicatedError
+        from cognite.neat.issues.errors._resources import ResourceNotDefinedError
+
+        reader = read_info_by_sheet.get("Properties", SpreadsheetRead())
+
+        if isinstance(caught_error, PropertyDefinitionDuplicatedError) and caught_error.location_name == "rows":
+            adjusted_row_number = (
+                tuple(
+                    reader.adjusted_row_number(row_no) if isinstance(row_no, int) else row_no
+                    for row_no in caught_error.locations or []
+                )
+                or None
+            )
+            # The error is frozen, so we have to use __setattr__ to change the row number
+            object.__setattr__(caught_error, "locations", adjusted_row_number)
+        elif isinstance(caught_error, RowError):
+            # Adjusting the row number to the actual row number in the spreadsheet
+            new_row = reader.adjusted_row_number(caught_error.row)
+            # The error is frozen, so we have to use __setattr__ to change the row number
+            object.__setattr__(caught_error, "row", new_row)
+        elif isinstance(caught_error, ResourceNotDefinedError):
+            if isinstance(caught_error.row_number, int) and caught_error.sheet_name == "Properties":
+                new_row = reader.adjusted_row_number(caught_error.row_number)
+                object.__setattr__(caught_error, "row_number", new_row)
+
+
+@dataclass(frozen=True)
+class DefaultPydanticError(NeatError, ValueError):
+    """{type}: {msg} [loc={loc}]"""
+
+    type: str
+    loc: tuple[int | str, ...]
+    msg: str
+
+    @classmethod
+    def from_pydantic_error(cls, error: ErrorDetails) -> "DefaultPydanticError":
+        return cls(
+            type=error["type"],
+            loc=error["loc"],
+            msg=error["msg"],
+        )
+
+    def as_message(self) -> str:
+        if self.loc and len(self.loc) == 1:
+            return f"{self.loc[0]} sheet: {self.msg}"
+        elif self.loc and len(self.loc) == 2:
+            return f"{self.loc[0]} sheet field/column <{self.loc[1]}>: {self.msg}"
+        else:
+            return self.msg
+
+
+@dataclass(frozen=True)
+class RowError(NeatError, ValueError):
+    """In {sheet_name}, row={row}, column={column}: {msg}. [type={type}, input_value={input}]"""
+
+    extra = "For further information visit {url}"
+
+    sheet_name: str
+    column: str
+    row: int
+    type: str
+    msg: str
+    input: Any
+    url: str | None = None
+
+    @classmethod
+    def from_pydantic_error(
+        cls,
+        error: ErrorDetails,
+        read_info_by_sheet: dict[str, SpreadsheetRead] | None = None,
+    ) -> Self:
+        sheet_name, _, row, column, *__ = error["loc"]
+        reader = (read_info_by_sheet or {}).get(str(sheet_name), SpreadsheetRead())
+        return cls(
+            sheet_name=str(sheet_name),
+            column=str(column),
+            row=reader.adjusted_row_number(int(row)),
+            type=error["type"],
+            msg=error["msg"],
+            input=error.get("input"),
+            url=str(url) if (url := error.get("url")) else None,
+        )
+
+    def as_message(self) -> str:
+        input_str = str(self.input) if self.input is not None else ""
+        input_str = input_str[:50] + "..." if len(input_str) > 50 else input_str
+        output = (
+            f"In {self.sheet_name}, row={self.row}, column={self.column}: {self.msg}. "
+            f"[type={self.type}, input_value={input_str}]"
+        )
+        if self.url:
+            output += f" For further information visit {self.url}"
+        return output
+
+
+@dataclass(frozen=True)
+class NeatWarning(NeatIssue, UserWarning):
+    """This is the base class for all warnings used in Neat."""
+
+    @classmethod
+    def from_warning(cls, warning: WarningMessage) -> "NeatWarning":
+        """Create a NeatWarning from a WarningMessage."""
+        return DefaultWarning.from_warning_message(warning)
+
+
+@dataclass(frozen=True)
+class DefaultWarning(NeatWarning):
+    """{category}: {warning}"""
+
+    extra = "Source: {source}"
+
+    warning: str
+    category: str
+    source: str | None = None
+
+    @classmethod
+    def from_warning_message(cls, warning: WarningMessage) -> NeatWarning:
+        if isinstance(warning.message, NeatWarning):
+            return warning.message
+
+        return cls(
+            warning=str(warning.message),
+            category=warning.category.__name__,
+            source=warning.source,
+        )
+
+    def as_message(self) -> str:
+        return str(self.warning)
+
+
+T_NeatIssue = TypeVar("T_NeatIssue", bound=NeatIssue)
+
+
+class NeatIssueList(UserList[T_NeatIssue], ABC):
+    """This is a generic list of NeatIssues."""
+
+    def __init__(self, issues: Sequence[T_NeatIssue] | None = None, title: str | None = None):
+        super().__init__(issues or [])
+        self.title = title
+
+    @property
+    def errors(self) -> Self:
+        """Return all the errors in this list."""
+        return type(self)([issue for issue in self if isinstance(issue, NeatError)])  # type: ignore[misc]
+
+    @property
+    def has_errors(self) -> bool:
+        """Return True if this list contains any errors."""
+        return any(isinstance(issue, NeatError) for issue in self)
+
+    @property
+    def warnings(self) -> Self:
+        """Return all the warnings in this list."""
+        return type(self)([issue for issue in self if isinstance(issue, NeatWarning)])  # type: ignore[misc]
+
+    def as_errors(self) -> ExceptionGroup:
+        """Return an ExceptionGroup with all the errors in this list."""
+        return ExceptionGroup(
+            "Operation failed",
+            [issue for issue in self if isinstance(issue, NeatError)],
+        )
+
+    def trigger_warnings(self) -> None:
+        """Trigger all warnings in this list."""
+        for warning in [issue for issue in self if isinstance(issue, NeatWarning)]:
+            warnings.warn(warning, stacklevel=2)
+
+    def to_pandas(self) -> pd.DataFrame:
+        """Return a pandas DataFrame representation of this list."""
+        return pd.DataFrame([issue.dump() for issue in self])
+
+    def _repr_html_(self) -> str | None:
+        return self.to_pandas()._repr_html_()  # type: ignore[operator]
+
+    def as_exception(self) -> "MultiValueError":
+        """Return a MultiValueError with all the errors in this list."""
+        return MultiValueError(self.errors)
+
+
+class MultiValueError(ValueError):
+    """This is a container for multiple errors.
+
+    It is used in the pydantic field_validator/model_validator to collect multiple errors, which
+    can then be caught in a try-except block and returned as an IssueList.
+
+    """
+
+    def __init__(self, errors: Sequence[NeatIssue]):
+        self.errors = list(errors)
+
+
+class IssueList(NeatIssueList[NeatIssue]):
+    """This is a list of NeatIssues."""
+
+    ...
+
+
+T_Cls = TypeVar("T_Cls")
+
+
+def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[type[T_Cls]]:
+    """Get all subclasses of a class."""
+    if include_base:
+        yield cls_
+    for s in cls_.__subclasses__():
+        yield s
+        yield from _get_subclasses(s, False)

cognite/neat/issues/errors/__init__.py

@@ -0,0 +1,72 @@
+from cognite.neat.issues._base import DefaultPydanticError, NeatError, RowError, _get_subclasses
+
+from ._external import (
+    AuthorizationError,
+    FileMissingRequiredFieldError,
+    FileNotAFileError,
+    FileNotFoundNeatError,
+    FileReadError,
+    FileTypeUnexpectedError,
+    NeatYamlError,
+)
+from ._general import NeatImportError, NeatValueError, RegexViolationError
+from ._properties import (
+    PropertyDefinitionDuplicatedError,
+    PropertyDefinitionError,
+    PropertyMappingDuplicatedError,
+    PropertyNotFoundError,
+    PropertyTypeNotSupportedError,
+)
+from ._resources import (
+    ResourceChangedError,
+    ResourceConvertionError,
+    ResourceCreationError,
+    ResourceDuplicatedError,
+    ResourceError,
+    ResourceMissingIdentifierError,
+    ResourceNotDefinedError,
+    ResourceNotFoundError,
+    ResourceRetrievalError,
+)
+from ._workflow import (
+    WorkflowConfigurationNotSetError,
+    WorkFlowMissingDataError,
+    WorkflowStepNotInitializedError,
+    WorkflowStepOutputError,
+)
+
+__all__ = [
+    "NeatError",
+    "NeatValueError",
+    "NeatImportError",
+    "RegexViolationError",
+    "AuthorizationError",
+    "NeatYamlError",
+    "FileReadError",
+    "ResourceCreationError",
+    "FileNotFoundNeatError",
+    "FileMissingRequiredFieldError",
+    "PropertyDefinitionError",
+    "PropertyTypeNotSupportedError",
+    "PropertyNotFoundError",
+    "PropertyDefinitionDuplicatedError",
+    "ResourceChangedError",
+    "ResourceDuplicatedError",
+    "ResourceRetrievalError",
+    "ResourceNotFoundError",
+    "ResourceError",
+    "ResourceNotDefinedError",
+    "ResourceMissingIdentifierError",
+    "ResourceConvertionError",
+    "WorkflowConfigurationNotSetError",
+    "WorkFlowMissingDataError",
+    "WorkflowStepNotInitializedError",
+    "WorkflowStepOutputError",
+    "FileTypeUnexpectedError",
+    "FileNotAFileError",
+    "DefaultPydanticError",
+    "PropertyMappingDuplicatedError",
+    "RowError",
+]
+
+_NEAT_ERRORS_BY_NAME = {error.__name__: error for error in _get_subclasses(NeatError, include_base=True)}

cognite/neat/issues/errors/_external.py

@@ -0,0 +1,67 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from yaml import YAMLError
+
+from cognite.neat.issues import NeatError
+
+
+@dataclass(frozen=True)
+class AuthorizationError(NeatError, RuntimeError):
+    """Missing authorization for {action}: {reason}"""
+
+    action: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class FileReadError(NeatError, RuntimeError):
+    """Error when reading file, {filepath}: {reason}"""
+
+    fix = "Is the {filepath} open in another program? Is the file corrupted?"
+    filepath: Path
+    reason: str
+
+
+@dataclass(frozen=True)
+class FileNotFoundNeatError(NeatError, FileNotFoundError):
+    """File {filepath} not found"""
+
+    fix = "Make sure to provide a valid file"
+    filepath: Path
+
+
+@dataclass(frozen=True)
+class FileMissingRequiredFieldError(NeatError, ValueError):
+    """Missing required {field_name} in {filepath}: {field}"""
+
+    filepath: Path
+    field_name: str
+    field: str
+
+
+@dataclass(frozen=True)
+class NeatYamlError(NeatError, YAMLError):
+    """Invalid YAML: {reason}"""
+
+    extra = "Expected format: {expected_format}"
+    fix = "Check if the file is a valid YAML file"
+
+    reason: str
+    expected_format: str | None = None
+
+
+@dataclass(frozen=True)
+class FileTypeUnexpectedError(NeatError, TypeError):
+    """Unexpected file type: {filepath}. Expected format: {expected_format}"""
+
+    filepath: Path
+    expected_format: frozenset[str]
+
+
+@dataclass(frozen=True)
+class FileNotAFileError(NeatError, FileNotFoundError):
+    """{filepath} is not a file"""
+
+    fix = "Make sure to provide a valid file"
+    filepath: Path

cognite/neat/issues/errors/_general.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+
+from cognite.neat.issues import NeatError
+
+
+@dataclass(frozen=True)
+class NeatValueError(NeatError, ValueError):
+    """{raw_message}"""
+
+    raw_message: str
+
+
+@dataclass(frozen=True)
+class RegexViolationError(NeatError, ValueError):
+    """Value, {value} failed regex, {regex}, validation. Make sure that the name follows the regex pattern."""
+
+    value: str
+    regex: str
+
+
+@dataclass(frozen=True)
+class NeatImportError(NeatError, ImportError):
+    """The functionality requires {module}. You can include it
+    in your neat installation with `pip install "cognite-neat[{neat_extra}]"`.
+    """
+
+    module: str
+    neat_extra: str

cognite/neat/issues/errors/_properties.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+from typing import Generic
+
+from cognite.neat.issues._base import ResourceType
+
+from ._resources import ResourceError, T_Identifier, T_ReferenceIdentifier
+
+
+@dataclass(frozen=True)
+class PropertyError(ResourceError[T_Identifier]):
+    """Base class for property errors {resource_type} with identifier {identifier}.{property_name}"""
+
+    property_name: str
+
+
+@dataclass(frozen=True)
+class PropertyNotFoundError(PropertyError, Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier} does not have a property {property_name}"""
+
+    extra = "referred to by {referred_type} {referred_by} does not exist"
+    fix = "Ensure the {resource_type} {identifier} has a {property_name} property"
+
+    referred_by: T_ReferenceIdentifier | None = None
+    referred_type: ResourceType | None = None
+
+
+@dataclass(frozen=True)
+class PropertyTypeNotSupportedError(PropertyError[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has a property {property_name}
+    of unsupported type {property_type}"""
+
+    property_type: str
+
+
+# This is a generic error that should be used sparingly
+@dataclass(frozen=True)
+class PropertyDefinitionError(PropertyError[T_Identifier]):
+    """Invalid property definition for {resource_type} {identifier}.{property_name}: {reason}"""
+
+    reason: str
+
+
+@dataclass(frozen=True)
+class PropertyDefinitionDuplicatedError(PropertyError[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has multiple definitions for the property {property_name}
+    with values {property_values}
+    """
+
+    extra = "in locations {locations} with name {location_name}"
+
+    property_values: frozenset[str | int | float | bool | None | tuple[str | int | float | bool | None, ...]]
+    locations: tuple[str | int, ...] | None = None
+    location_name: str | None = None
+
+
+@dataclass(frozen=True)
+class PropertyMappingDuplicatedError(PropertyError[T_Identifier], Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier}.{property_name} is mapped to by: {mappings}. Ensure
+    that only one {mapping_type} maps to {resource_type} {identifier}.{property_name}"""
+
+    mappings: frozenset[T_ReferenceIdentifier]
+    mapping_type: ResourceType