cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

cognite/neat/issues/_base.py

@@ -1,15 +1,21 @@
 import sys
 import warnings
-from abc import ABC, abstractmethod
+from abc import ABC
 from collections import UserList
-from collections.abc import Sequence
-from dataclasses import dataclass
+from collections.abc import Collection, Hashable, Iterable, Sequence
+from dataclasses import dataclass, fields
 from functools import total_ordering
-from typing import Any, ClassVar, TypeVar
+from pathlib import Path
+from types import UnionType
+from typing import Any, ClassVar, Literal, TypeAlias, TypeVar, get_args, get_origin
 from warnings import WarningMessage
 
 import pandas as pd
-from pydantic_core import ErrorDetails, PydanticCustomError
+from cognite.client.data_classes.data_modeling import ContainerId, ViewId
+from pydantic_core import ErrorDetails
+
+from cognite.neat.utils.spreadsheet import SpreadsheetRead
+from cognite.neat.utils.text import humanize_collection, to_camel, to_snake
 
 if sys.version_info < (3, 11):
     from exceptiongroup import ExceptionGroup
@@ -27,55 +33,151 @@ __all__ = [
     "MultiValueError",
 ]
 
+T_Identifier = TypeVar("T_Identifier", bound=Hashable)
+
+T_ReferenceIdentifier = TypeVar("T_ReferenceIdentifier", bound=Hashable)
+
+ResourceType: TypeAlias = (
+    Literal[
+        "view",
+        "container",
+        "view property",
+        "container property",
+        "space",
+        "class",
+        "asset",
+        "relationship",
+        "data model",
+        "edge",
+        "node",
+        "enum collection",
+        "unknown",
+    ]
+    # String to handle all unknown types in different importers.
+    | str
+)
+
 
 @total_ordering
 @dataclass(frozen=True)
-class NeatIssue(ABC):
+class NeatIssue:
     """This is the base class for all exceptions and warnings (issues) used in Neat."""
 
-    description: ClassVar[str]
     extra: ClassVar[str | None] = None
     fix: ClassVar[str | None] = None
 
-    def message(self) -> str:
-        """Return a human-readable message for the issue.
-
-        This is the default implementation, which returns the description.
-        It is recommended to override this method in subclasses with a more
-        specific message.
-        """
-        return self.__doc__ or "Missing"
+    def as_message(self) -> str:
+        """Return a human-readable message for the issue."""
+        template = self.__doc__
+        if not template:
+            return "Missing"
+        variables, has_all_optional = self._get_variables()
+
+        msg = template.format(**variables)
+        if self.extra and has_all_optional:
+            msg += "\n" + self.extra.format(**variables)
+        if self.fix:
+            msg += f"\nFix: {self.fix.format(**variables)}"
+        name = type(self).__name__
+        return f"{name}: {msg}"
+
+    def _get_variables(self) -> tuple[dict[str, str], bool]:
+        variables: dict[str, str] = {}
+        has_all_optional = True
+        for name, var_ in vars(self).items():
+            if var_ is None:
+                has_all_optional = False
+            elif isinstance(var_, str):
+                variables[name] = var_
+            elif isinstance(var_, Path):
+                variables[name] = var_.as_posix()
+            elif isinstance(var_, Collection):
+                variables[name] = humanize_collection(var_)
+            else:
+                variables[name] = repr(var_)
+        return variables, has_all_optional
 
-    @abstractmethod
     def dump(self) -> dict[str, Any]:
         """Return a dictionary representation of the issue."""
-        raise NotImplementedError()
+        variables = vars(self)
+        output = {to_camel(key): self._dump_value(value) for key, value in variables.items() if value is not None}
+        output["NeatIssue"] = type(self).__name__
+        return output
+
+    @classmethod
+    def _dump_value(cls, value: Any) -> list | int | bool | float | str | dict:
+        if isinstance(value, str | int | bool | float):
+            return value
+        elif isinstance(value, frozenset):
+            return [cls._dump_value(item) for item in value]
+        elif isinstance(value, Path):
+            return value.as_posix()
+        elif isinstance(value, tuple):
+            return [cls._dump_value(item) for item in value]
+        elif isinstance(value, ViewId | ContainerId):
+            return value.dump(camel_case=True, include_type=True)
+        raise ValueError(f"Unsupported type: {type(value)}")
+
+    @classmethod
+    def load(cls, data: dict[str, Any]) -> "NeatIssue":
+        """Create an instance of the issue from a dictionary."""
+        from cognite.neat.issues.errors import _NEAT_ERRORS_BY_NAME, NeatValueError
+        from cognite.neat.issues.warnings import _NEAT_WARNINGS_BY_NAME
+
+        if "NeatIssue" not in data:
+            raise NeatValueError("The data does not contain a NeatIssue key.")
+        issue_type = data.pop("NeatIssue")
+        args = {to_snake(key): value for key, value in data.items()}
+        if issue_type in _NEAT_ERRORS_BY_NAME:
+            return cls._load_values(_NEAT_ERRORS_BY_NAME[issue_type], args)
+        elif issue_type in _NEAT_WARNINGS_BY_NAME:
+            return cls._load_values(_NEAT_WARNINGS_BY_NAME[issue_type], args)
+        else:
+            raise NeatValueError(f"Unknown issue type: {issue_type}")
+
+    @classmethod
+    def _load_values(cls, neat_issue_cls: "type[NeatIssue]", data: dict[str, Any]) -> "NeatIssue":
+        args: dict[str, Any] = {}
+        for f in fields(neat_issue_cls):
+            if f.name not in data:
+                continue
+            value = data[f.name]
+            args[f.name] = cls._load_value(f.type, value)
+        return neat_issue_cls(**args)
+
+    @classmethod
+    def _load_value(cls, type_: type, value: Any) -> Any:
+        if isinstance(type_, UnionType) or get_origin(type_) is UnionType:
+            args = get_args(type_)
+            return cls._load_value(args[0], value)
+        elif type_ is frozenset or get_origin(type_) is frozenset:
+            subtype = get_args(type_)[0]
+            return frozenset(cls._load_value(subtype, item) for item in value)
+        elif type_ is Path:
+            return Path(value)
+        elif type_ is tuple or get_origin(type_) is tuple:
+            subtype = get_args(type_)[0]
+            return tuple(cls._load_value(subtype, item) for item in value)
+        elif type_ is ViewId:
+            return ViewId.load(value)
+        elif type_ is ContainerId:
+            return ContainerId.load(value)
+        return value
 
     def __lt__(self, other: "NeatIssue") -> bool:
         if not isinstance(other, NeatIssue):
             return NotImplemented
-        return (type(self).__name__, self.message()) < (type(other).__name__, other.message())
+        return (type(self).__name__, self.as_message()) < (type(other).__name__, other.as_message())
 
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, NeatIssue):
             return NotImplemented
-        return (type(self).__name__, self.message()) == (type(other).__name__, other.message())
+        return (type(self).__name__, self.as_message()) == (type(other).__name__, other.as_message())
 
 
 @dataclass(frozen=True)
-class NeatError(NeatIssue, ABC):
-    def dump(self) -> dict[str, Any]:
-        return {"errorType": type(self).__name__}
-
-    def as_exception(self) -> ValueError:
-        return ValueError(self.message())
-
-    def as_pydantic_exception(self) -> PydanticCustomError:
-        return PydanticCustomError(
-            type(self).__name__,
-            self.message(),
-            dict(description=self.__doc__, fix=self.fix),
-        )
+class NeatError(NeatIssue, Exception):
+    """This is the base class for all exceptions (errors) used in Neat."""
 
     @classmethod
     def from_pydantic_errors(cls, errors: list[ErrorDetails], **kwargs) -> "list[NeatError]":
@@ -84,23 +186,60 @@ class NeatError(NeatIssue, ABC):
         This is intended to be overridden in subclasses to handle specific error types.
         """
         all_errors: list[NeatError] = []
+        read_info_by_sheet = kwargs.get("read_info_by_sheet")
+
         for error in errors:
-            if isinstance(ctx := error.get("ctx"), dict) and isinstance(
-                multi_error := ctx.get("error"), MultiValueError
-            ):
+            ctx = error.get("ctx")
+            if isinstance(ctx, dict) and isinstance(multi_error := ctx.get("error"), MultiValueError):
+                if read_info_by_sheet:
+                    for caught_error in multi_error.errors:
+                        cls._adjust_row_numbers(caught_error, read_info_by_sheet)  # type: ignore[arg-type]
                 all_errors.extend(multi_error.errors)  # type: ignore[arg-type]
+            elif isinstance(ctx, dict) and isinstance(single_error := ctx.get("error"), NeatError):
+                if read_info_by_sheet:
+                    cls._adjust_row_numbers(single_error, read_info_by_sheet)
+                all_errors.append(single_error)
+            elif len(error["loc"]) >= 4 and read_info_by_sheet:
+                all_errors.append(RowError.from_pydantic_error(error, read_info_by_sheet))
             else:
                 all_errors.append(DefaultPydanticError.from_pydantic_error(error))
         return all_errors
 
+    @staticmethod
+    def _adjust_row_numbers(caught_error: "NeatError", read_info_by_sheet: dict[str, SpreadsheetRead]) -> None:
+        from cognite.neat.issues.errors._properties import PropertyDefinitionDuplicatedError
+        from cognite.neat.issues.errors._resources import ResourceNotDefinedError
+
+        reader = read_info_by_sheet.get("Properties", SpreadsheetRead())
+
+        if isinstance(caught_error, PropertyDefinitionDuplicatedError) and caught_error.location_name == "rows":
+            adjusted_row_number = (
+                tuple(
+                    reader.adjusted_row_number(row_no) if isinstance(row_no, int) else row_no
+                    for row_no in caught_error.locations or []
+                )
+                or None
+            )
+            # The error is frozen, so we have to use __setattr__ to change the row number
+            object.__setattr__(caught_error, "locations", adjusted_row_number)
+        elif isinstance(caught_error, RowError):
+            # Adjusting the row number to the actual row number in the spreadsheet
+            new_row = reader.adjusted_row_number(caught_error.row)
+            # The error is frozen, so we have to use __setattr__ to change the row number
+            object.__setattr__(caught_error, "row", new_row)
+        elif isinstance(caught_error, ResourceNotDefinedError):
+            if isinstance(caught_error.row_number, int) and caught_error.sheet_name == "Properties":
+                new_row = reader.adjusted_row_number(caught_error.row_number)
+                object.__setattr__(caught_error, "row_number", new_row)
+
 
 @dataclass(frozen=True)
-class DefaultPydanticError(NeatError):
+class DefaultPydanticError(NeatError, ValueError):
+    """{type}: {msg} [loc={loc}]"""
+
     type: str
     loc: tuple[int | str, ...]
     msg: str
-    input: Any
-    ctx: dict[str, Any] | None
 
     @classmethod
     def from_pydantic_error(cls, error: ErrorDetails) -> "DefaultPydanticError":
@@ -108,20 +247,9 @@ class DefaultPydanticError(NeatError):
             type=error["type"],
             loc=error["loc"],
             msg=error["msg"],
-            input=error.get("input"),
-            ctx=error.get("ctx"),
         )
 
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["type"] = self.type
-        output["loc"] = self.loc
-        output["msg"] = self.msg
-        output["input"] = self.input
-        output["ctx"] = self.ctx
-        return output
-
-    def message(self) -> str:
+    def as_message(self) -> str:
         if self.loc and len(self.loc) == 1:
             return f"{self.loc[0]} sheet: {self.msg}"
         elif self.loc and len(self.loc) == 2:
@@ -131,30 +259,68 @@ class DefaultPydanticError(NeatError):
 
 
 @dataclass(frozen=True)
-class NeatWarning(NeatIssue, ABC, UserWarning):
-    def dump(self) -> dict[str, Any]:
-        return {"warningType": type(self).__name__}
+class RowError(NeatError, ValueError):
+    """In {sheet_name}, row={row}, column={column}: {msg}. [type={type}, input_value={input}]"""
+
+    extra = "For further information visit {url}"
+
+    sheet_name: str
+    column: str
+    row: int
+    type: str
+    msg: str
+    input: Any
+    url: str | None = None
+
+    @classmethod
+    def from_pydantic_error(
+        cls,
+        error: ErrorDetails,
+        read_info_by_sheet: dict[str, SpreadsheetRead] | None = None,
+    ) -> Self:
+        sheet_name, _, row, column, *__ = error["loc"]
+        reader = (read_info_by_sheet or {}).get(str(sheet_name), SpreadsheetRead())
+        return cls(
+            sheet_name=str(sheet_name),
+            column=str(column),
+            row=reader.adjusted_row_number(int(row)),
+            type=error["type"],
+            msg=error["msg"],
+            input=error.get("input"),
+            url=str(url) if (url := error.get("url")) else None,
+        )
+
+    def as_message(self) -> str:
+        input_str = str(self.input) if self.input is not None else ""
+        input_str = input_str[:50] + "..." if len(input_str) > 50 else input_str
+        output = (
+            f"In {self.sheet_name}, row={self.row}, column={self.column}: {self.msg}. "
+            f"[type={self.type}, input_value={input_str}]"
+        )
+        if self.url:
+            output += f" For further information visit {self.url}"
+        return output
+
+
+@dataclass(frozen=True)
+class NeatWarning(NeatIssue, UserWarning):
+    """This is the base class for all warnings used in Neat."""
 
     @classmethod
     def from_warning(cls, warning: WarningMessage) -> "NeatWarning":
+        """Create a NeatWarning from a WarningMessage."""
         return DefaultWarning.from_warning_message(warning)
 
 
 @dataclass(frozen=True)
 class DefaultWarning(NeatWarning):
-    description = "A warning was raised during validation."
-    fix = "No fix is available."
+    """{category}: {warning}"""
 
-    warning: str | Warning
-    category: type[Warning]
-    source: str | None = None
+    extra = "Source: {source}"
 
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["msg"] = str(self.warning)
-        output["category"] = self.category.__name__
-        output["source"] = self.source
-        return output
+    warning: str
+    category: str
+    source: str | None = None
 
     @classmethod
     def from_warning_message(cls, warning: WarningMessage) -> NeatWarning:
@@ -162,52 +328,61 @@ class DefaultWarning(NeatWarning):
             return warning.message
 
         return cls(
-            warning=warning.message,
-            category=warning.category,
+            warning=str(warning.message),
+            category=warning.category.__name__,
             source=warning.source,
         )
 
-    def message(self) -> str:
+    def as_message(self) -> str:
         return str(self.warning)
 
 
 T_NeatIssue = TypeVar("T_NeatIssue", bound=NeatIssue)
 
 
-class NeatIssueList(UserList[T_NeatIssue], ABC):
+class NeatIssueList(UserList[T_NeatIssue], Sequence[T_NeatIssue], ABC):
+    """This is a generic list of NeatIssues."""
+
     def __init__(self, issues: Sequence[T_NeatIssue] | None = None, title: str | None = None):
         super().__init__(issues or [])
         self.title = title
 
     @property
     def errors(self) -> Self:
+        """Return all the errors in this list."""
         return type(self)([issue for issue in self if isinstance(issue, NeatError)])  # type: ignore[misc]
 
     @property
     def has_errors(self) -> bool:
+        """Return True if this list contains any errors."""
         return any(isinstance(issue, NeatError) for issue in self)
 
     @property
     def warnings(self) -> Self:
+        """Return all the warnings in this list."""
         return type(self)([issue for issue in self if isinstance(issue, NeatWarning)])  # type: ignore[misc]
 
     def as_errors(self) -> ExceptionGroup:
+        """Return an ExceptionGroup with all the errors in this list."""
         return ExceptionGroup(
             "Operation failed",
-            [ValueError(issue.message()) for issue in self if isinstance(issue, NeatError)],
+            [issue for issue in self if isinstance(issue, NeatError)],
         )
 
     def trigger_warnings(self) -> None:
+        """Trigger all warnings in this list."""
         for warning in [issue for issue in self if isinstance(issue, NeatWarning)]:
             warnings.warn(warning, stacklevel=2)
 
     def to_pandas(self) -> pd.DataFrame:
+        """Return a pandas DataFrame representation of this list."""
         return pd.DataFrame([issue.dump() for issue in self])
 
     def _repr_html_(self) -> str | None:
         return self.to_pandas()._repr_html_()  # type: ignore[operator]
 
     def as_exception(self) -> "MultiValueError":
+        """Return a MultiValueError with all the errors in this list."""
         return MultiValueError(self.errors)
 
 
@@ -219,8 +394,23 @@ class MultiValueError(ValueError):
 
     """
 
-    def __init__(self, errors: Sequence[T_NeatIssue]):
+    def __init__(self, errors: Sequence[NeatIssue]):
         self.errors = list(errors)
 
 
-class IssueList(NeatIssueList[NeatIssue]): ...
+class IssueList(NeatIssueList[NeatIssue]):
+    """This is a list of NeatIssues."""
+
+    ...
+
+
+T_Cls = TypeVar("T_Cls")
+
+
+def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[type[T_Cls]]:
+    """Get all subclasses of a class."""
+    if include_base:
+        yield cls_
+    for s in cls_.__subclasses__():
+        yield s
+        yield from _get_subclasses(s, False)

cognite/neat/issues/errors/__init__.py

@@ -0,0 +1,73 @@
+from cognite.neat.issues._base import DefaultPydanticError, NeatError, RowError, _get_subclasses
+
+from ._external import (
+    AuthorizationError,
+    FileMissingRequiredFieldError,
+    FileNotAFileError,
+    FileNotFoundNeatError,
+    FileReadError,
+    FileTypeUnexpectedError,
+    NeatYamlError,
+)
+from ._general import NeatImportError, NeatTypeError, NeatValueError, RegexViolationError
+from ._properties import (
+    PropertyDefinitionDuplicatedError,
+    PropertyDefinitionError,
+    PropertyMappingDuplicatedError,
+    PropertyNotFoundError,
+    PropertyTypeNotSupportedError,
+)
+from ._resources import (
+    ResourceChangedError,
+    ResourceConvertionError,
+    ResourceCreationError,
+    ResourceDuplicatedError,
+    ResourceError,
+    ResourceMissingIdentifierError,
+    ResourceNotDefinedError,
+    ResourceNotFoundError,
+    ResourceRetrievalError,
+)
+from ._workflow import (
+    WorkflowConfigurationNotSetError,
+    WorkFlowMissingDataError,
+    WorkflowStepNotInitializedError,
+    WorkflowStepOutputError,
+)
+
+__all__ = [
+    "NeatError",
+    "NeatValueError",
+    "NeatImportError",
+    "RegexViolationError",
+    "AuthorizationError",
+    "NeatYamlError",
+    "FileReadError",
+    "ResourceCreationError",
+    "FileNotFoundNeatError",
+    "FileMissingRequiredFieldError",
+    "PropertyDefinitionError",
+    "PropertyTypeNotSupportedError",
+    "PropertyNotFoundError",
+    "PropertyDefinitionDuplicatedError",
+    "ResourceChangedError",
+    "ResourceDuplicatedError",
+    "ResourceRetrievalError",
+    "ResourceNotFoundError",
+    "ResourceError",
+    "ResourceNotDefinedError",
+    "ResourceMissingIdentifierError",
+    "ResourceConvertionError",
+    "WorkflowConfigurationNotSetError",
+    "WorkFlowMissingDataError",
+    "WorkflowStepNotInitializedError",
+    "WorkflowStepOutputError",
+    "FileTypeUnexpectedError",
+    "FileNotAFileError",
+    "DefaultPydanticError",
+    "PropertyMappingDuplicatedError",
+    "RowError",
+    "NeatTypeError",
+]
+
+_NEAT_ERRORS_BY_NAME = {error.__name__: error for error in _get_subclasses(NeatError, include_base=True)}

cognite/neat/issues/errors/_external.py

@@ -0,0 +1,67 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from yaml import YAMLError
+
+from cognite.neat.issues import NeatError
+
+
+@dataclass(frozen=True)
+class AuthorizationError(NeatError, RuntimeError):
+    """Missing authorization for {action}: {reason}"""
+
+    action: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class FileReadError(NeatError, RuntimeError):
+    """Error when reading file, {filepath}: {reason}"""
+
+    fix = "Is the {filepath} open in another program? Is the file corrupted?"
+    filepath: Path
+    reason: str
+
+
+@dataclass(frozen=True)
+class FileNotFoundNeatError(NeatError, FileNotFoundError):
+    """File {filepath} not found"""
+
+    fix = "Make sure to provide a valid file"
+    filepath: Path
+
+
+@dataclass(frozen=True)
+class FileMissingRequiredFieldError(NeatError, ValueError):
+    """Missing required {field_name} in {filepath}: {field}"""
+
+    filepath: Path
+    field_name: str
+    field: str
+
+
+@dataclass(frozen=True)
+class NeatYamlError(NeatError, YAMLError):
+    """Invalid YAML: {reason}"""
+
+    extra = "Expected format: {expected_format}"
+    fix = "Check if the file is a valid YAML file"
+
+    reason: str
+    expected_format: str | None = None
+
+
+@dataclass(frozen=True)
+class FileTypeUnexpectedError(NeatError, TypeError):
+    """Unexpected file type: {filepath}. Expected format: {expected_format}"""
+
+    filepath: Path
+    expected_format: frozenset[str]
+
+
+@dataclass(frozen=True)
+class FileNotAFileError(NeatError, FileNotFoundError):
+    """{filepath} is not a file"""
+
+    fix = "Make sure to provide a valid file"
+    filepath: Path

cognite/neat/issues/errors/_general.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+
+from cognite.neat.issues import NeatError
+
+
+@dataclass(frozen=True)
+class NeatValueError(NeatError, ValueError):
+    """{raw_message}"""
+
+    raw_message: str
+
+
+@dataclass(frozen=True)
+class NeatTypeError(NeatError, TypeError):
+    """{raw_message}"""
+
+    raw_message: str
+
+
+@dataclass(frozen=True)
+class RegexViolationError(NeatError, ValueError):
+    """Value, {value} failed regex, {regex}, validation. Make sure that the name follows the regex pattern."""
+
+    value: str
+    regex: str
+
+
+@dataclass(frozen=True)
+class NeatImportError(NeatError, ImportError):
+    """The functionality requires {module}. You can include it
+    in your neat installation with `pip install "cognite-neat[{neat_extra}]"`.
+    """
+
+    module: str
+    neat_extra: str