cognite-neat 0.109.4__py3-none-any.whl → 0.110.0__py3-none-any.whl

cognite/neat/_issues/_contextmanagers.py

@@ -0,0 +1,46 @@
+import warnings
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from pydantic import ValidationError
+
+from cognite.neat._utils.spreadsheet import SpreadsheetRead
+
+from ._base import IssueList, MultiValueError, NeatError
+from ._factory import from_pydantic_errors, from_warning
+
+
+@contextmanager
+def catch_warnings() -> Iterator[IssueList]:
+    """Catch warnings and append them to the issues list."""
+    issues = IssueList()
+    with warnings.catch_warnings(record=True) as warning_logger:
+        warnings.simplefilter("always")
+        try:
+            yield issues
+        finally:
+            if warning_logger:
+                issues.extend([from_warning(warning) for warning in warning_logger])
+
+
+@contextmanager
+def catch_issues(read_info_by_sheet: dict[str, SpreadsheetRead] | None = None) -> Iterator[IssueList]:
+    """This is an internal help function to handle issues and warnings.
+
+    Args:
+        read_info_by_sheet (dict[str, SpreadsheetRead]): The read information by sheet. This is used to adjust
+            the row numbers in the errors/warnings.
+
+    Returns:
+        IssueList: The list of issues.
+
+    """
+    with catch_warnings() as issues:
+        try:
+            yield issues
+        except ValidationError as e:
+            issues.extend(from_pydantic_errors(e.errors(), read_info_by_sheet))
+        except NeatError as single:
+            issues.append(single)
+        except MultiValueError as multi:
+            issues.extend(multi.errors)

cognite/neat/_issues/_factory.py

@@ -0,0 +1,61 @@
+from typing import cast
+from warnings import WarningMessage
+
+from pydantic_core import ErrorDetails
+
+from cognite.neat._issues._base import NeatError, NeatWarning
+from cognite.neat._utils.spreadsheet import SpreadsheetRead
+
+from .errors import NeatValueError, SpreadsheetError
+from .warnings import NeatValueWarning
+
+
+def from_pydantic_errors(
+    errors: list[ErrorDetails], read_info_by_sheet: dict[str, SpreadsheetRead] | None = None
+) -> list[NeatError]:
+    read_info_by_sheet = read_info_by_sheet or {}
+    return [
+        _from_pydantic_error(error, read_info_by_sheet)
+        for error in errors
+        # Skip the error for SheetList, as it is not relevant for the user. This is an
+        # internal class used to have helper methods for a lists as .to_pandas()
+        if not (error["type"] == "is_instance_of" and error["loc"][1] == "is-instance[SheetList]")
+    ]
+
+
+def from_warning(warning: WarningMessage) -> NeatWarning:
+    if isinstance(warning.message, NeatWarning):
+        return warning.message
+    message = f"{warning.category.__name__}: {warning.message!s}"
+    if warning.source:
+        message += f" Source: {warning.source}"
+    return NeatValueWarning(message)
+
+
+def _from_pydantic_error(error: ErrorDetails, read_info_by_sheet: dict[str, SpreadsheetRead]) -> NeatError:
+    neat_error = _create_neat_value_error(error)
+    location = error["loc"]
+    return SpreadsheetError.create(location, neat_error, read_info_by_sheet.get(cast(str, location[0])))
+
+
+def _create_neat_value_error(error: ErrorDetails) -> NeatValueError:
+    if (ctx := error.get("ctx")) and (neat_error := ctx.get("error")) and isinstance(neat_error, NeatError):
+        # Is already a NeatError
+        return neat_error
+    return _pydantic_to_neat_error(error)
+
+
+def _pydantic_to_neat_error(error: ErrorDetails) -> NeatValueError:
+    error_type = error["type"]
+    input_value = error["input"]
+    match error_type:
+        # See https://docs.pydantic.dev/latest/errors/validation_errors/ for all possible error types:
+        case error_type if error_type.endswith("_type") | error_type.endswith("_parsing"):
+            if input_value is None:
+                return NeatValueError("value is missing.")
+            expected_type = error_type.removesuffix("_type").removesuffix("_parsing")
+            return NeatValueError(f"Expected a {expected_type} type, got {input_value!r}")
+        case _:
+            # The above cases overwrite the human-readable message from pydantic.
+            # Motivation for overwriting is that pydantic is developer-oriented and while neat is SME-oriented.
+            return NeatValueError(f"{error['msg']} got '{input_value}'")

cognite/neat/_issues/errors/__init__.py

@@ -1,4 +1,4 @@
-from cognite.neat._issues._base import DefaultPydanticError, NeatError, RowError, _get_subclasses
+from cognite.neat._issues._base import NeatError, _get_subclasses
 
 from ._external import (
     AuthorizationError,
@@ -31,12 +31,23 @@ from ._resources import (
     ResourceNotFoundError,
     ResourceRetrievalError,
 )
-from ._wrapper import MetadataValueError
+from ._wrapper import (
+    ClassValueError,
+    ContainerValueError,
+    EnumValueError,
+    MetadataValueError,
+    NodeValueError,
+    PropertyValueError,
+    SpreadsheetError,
+    ViewValueError,
+)
 
 __all__ = [
     "AuthorizationError",
     "CDFMissingClientError",
-    "DefaultPydanticError",
+    "ClassValueError",
+    "ContainerValueError",
+    "EnumValueError",
     "FileMissingRequiredFieldError",
     "FileNotAFileError",
     "FileNotFoundNeatError",
@@ -48,12 +59,14 @@ __all__ = [
     "NeatTypeError",
     "NeatValueError",
     "NeatYamlError",
+    "NodeValueError",
     "OxigraphStorageLockedError",
     "PropertyDefinitionDuplicatedError",
     "PropertyDefinitionError",
     "PropertyMappingDuplicatedError",
     "PropertyNotFoundError",
     "PropertyTypeNotSupportedError",
+    "PropertyValueError",
     "RegexViolationError",
     "ResourceChangedError",
     "ResourceConversionError",
@@ -65,7 +78,8 @@ __all__ = [
     "ResourceNotFoundError",
     "ResourceRetrievalError",
     "ReversedConnectionNotFeasibleError",
-    "RowError",
+    "SpreadsheetError",
+    "ViewValueError",
 ]
 
 _NEAT_ERRORS_BY_NAME = {error.__name__: error for error in _get_subclasses(NeatError, include_base=True)}

cognite/neat/_issues/errors/_wrapper.py

@@ -1,11 +1,89 @@
+from abc import ABC
 from dataclasses import dataclass
+from typing import ClassVar, cast
 
 from cognite.neat._issues import NeatError
+from cognite.neat._utils.spreadsheet import SpreadsheetRead
 
 
 @dataclass(unsafe_hash=True)
-class MetadataValueError(NeatError, ValueError):
-    """Field {field_name} - {error}"""
+class SpreadsheetError(NeatError, ValueError, ABC):
+    """In row {row}: {error}"""
 
-    field_name: str
+    _name: ClassVar[str] = ""
     error: NeatError
+
+    @classmethod
+    def create(
+        cls, location: tuple[int | str, ...], error: NeatError, spreadsheet: SpreadsheetRead | None = None
+    ) -> "SpreadsheetError":
+        spreadsheet_name = cast(str, location[0])
+        if spreadsheet_name not in ERROR_CLS_BY_SPREADSHEET_NAME:
+            # This happens for the metadata sheet, which are individual fields
+            return MetadataValueError(error, field_name=spreadsheet_name)
+
+        error_cls = ERROR_CLS_BY_SPREADSHEET_NAME[spreadsheet_name]
+        row, column = cast(tuple[int, str], location[2:4])
+
+        if spreadsheet:
+            row = spreadsheet.adjusted_row_number(row)
+
+        return error_cls(
+            row=row,
+            error=error,
+            column=column,
+        )
+
+
+@dataclass(unsafe_hash=True)
+class SpreadsheetListError(SpreadsheetError, ABC):
+    """In row {row}, column '{column}': {error}"""
+
+    row: int
+    column: str
+
+
+@dataclass(unsafe_hash=True)
+class MetadataValueError(SpreadsheetError):
+    """In field {field_name}: {error}"""
+
+    _type: ClassVar[str] = "Metadata"
+    field_name: str
+
+
+@dataclass(unsafe_hash=True)
+class ViewValueError(SpreadsheetListError):
+    _name = "Views"
+
+
+@dataclass(unsafe_hash=True)
+class ContainerValueError(SpreadsheetListError):
+    _name = "Containers"
+
+
+@dataclass(unsafe_hash=True)
+class PropertyValueError(SpreadsheetListError):
+    _name = "Properties"
+
+
+@dataclass(unsafe_hash=True)
+class ClassValueError(SpreadsheetListError):
+    _name = "Classes"
+
+
+@dataclass(unsafe_hash=True)
+class EnumValueError(SpreadsheetListError):
+    _name = "Enum"
+
+
+@dataclass(unsafe_hash=True)
+class NodeValueError(SpreadsheetListError):
+    _name = "Nodes"
+
+
+ERROR_CLS_BY_SPREADSHEET_NAME = {cls_._name: cls_ for cls_ in SpreadsheetListError.__subclasses__()}
+
+# Efficient way to set docstring for all classes
+for _cls in ERROR_CLS_BY_SPREADSHEET_NAME.values():
+    _cls.__doc__ = SpreadsheetListError.__doc__
+del _cls

cognite/neat/_issues/formatters.py

@@ -3,7 +3,7 @@ import xml.etree.ElementTree as ET
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from ._base import NeatError, NeatIssueList, NeatWarning
+from ._base import IssueList, NeatError, NeatWarning
 
 __all__ = ["FORMATTER_BY_NAME", "BasicHTML", "Formatter"]
 
@@ -13,14 +13,14 @@ class Formatter(ABC):
     default_file_prefix: str = "validation_report"
 
     @abstractmethod
-    def create_report(self, issues: NeatIssueList) -> str:
+    def create_report(self, issues: IssueList) -> str:
         raise NotImplementedError()
 
     @property
     def default_file_name(self) -> str:
         return f"{self.default_file_prefix}_{type(self).__name__.lower()}{self.file_suffix}"
 
-    def write_to_file(self, issues: NeatIssueList, file_or_dir_path: Path | None = None) -> None:
+    def write_to_file(self, issues: IssueList, file_or_dir_path: Path | None = None) -> None:
         if file_or_dir_path is None:
             file_or_dir_path = Path(self.default_file_name)
         elif file_or_dir_path.is_dir():
@@ -41,7 +41,7 @@ class BasicHTML(Formatter):
         self._doc = ET.Element("html")
         self._body = ET.SubElement(self._doc, "body")
 
-    def create_report(self, issues: NeatIssueList) -> str:
+    def create_report(self, issues: IssueList) -> str:
         errors = [issue for issue in issues if isinstance(issue, NeatError)]
         warnings_ = [issue for issue in issues if isinstance(issue, NeatWarning)]
         self._doc.clear()

cognite/neat/_issues/warnings/__init__.py

@@ -2,7 +2,7 @@
 conflicts with the built-in Python warnings module. However, it is expected to always be used in an absolute
 import, and should thus not cause a naming conflict."""
 
-from cognite.neat._issues._base import DefaultWarning, NeatWarning, _get_subclasses
+from cognite.neat._issues._base import NeatWarning, _get_subclasses
 
 from . import user_modeling
 from ._external import (
@@ -29,6 +29,7 @@ from ._properties import (
     PropertyDataTypeConversionWarning,
     PropertyDefinitionDuplicatedWarning,
     PropertyDirectRelationLimitWarning,
+    PropertyMultipleValueWarning,
     PropertyNotFoundWarning,
     PropertyOverwritingWarning,
     PropertyTypeNotSupportedWarning,
@@ -49,7 +50,6 @@ __all__ = [
     "CDFAuthWarning",
     "CDFMaxIterationsWarning",
     "CDFNotSupportedWarning",
-    "DefaultWarning",
     "FileItemNotSupportedWarning",
     "FileMissingRequiredFieldWarning",
     "FileReadWarning",
@@ -65,6 +65,7 @@ __all__ = [
     "PropertyDataTypeConversionWarning",
     "PropertyDefinitionDuplicatedWarning",
     "PropertyDirectRelationLimitWarning",
+    "PropertyMultipleValueWarning",
     "PropertyNotFoundWarning",
     "PropertyOverwritingWarning",
     "PropertyTypeNotSupportedWarning",

cognite/neat/_issues/warnings/_properties.py

@@ -80,3 +80,11 @@ class PropertyDirectRelationLimitWarning(PropertyWarning[T_Identifier]):
     resource_type = "view"
 
     limit: int = DMS_DIRECT_RELATION_LIST_LIMIT
+
+
+@dataclass(unsafe_hash=True)
+class PropertyMultipleValueWarning(PropertyWarning[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has a property {property_name} with multiple values.
+    Selecting the first value {value}, the rest will be ignored."""
+
+    value: str

cognite/neat/_rules/_constants.py

@@ -185,3 +185,12 @@ class _Patterns:
 
 
 PATTERNS = _Patterns()
+
+
+def get_internal_properties() -> set[str]:
+    return {
+        "physical",
+        "logical",
+        "conceptual",
+        "Neat ID",
+    }

cognite/neat/_rules/_shared.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, Generic, TypeAlias, TypeVar
+from typing import Generic, TypeAlias, TypeVar
 
 from cognite.neat._rules.models import (
     DMSRules,
@@ -7,6 +7,7 @@ from cognite.neat._rules.models import (
 )
 from cognite.neat._rules.models.dms._rules_input import DMSInputRules
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
+from cognite.neat._utils.spreadsheet import SpreadsheetRead
 
 VerifiedRules: TypeAlias = InformationRules | DMSRules
 
@@ -20,7 +21,7 @@ class ReadRules(Generic[T_InputRules]):
     """This represents a rules that has been read."""
 
     rules: T_InputRules | None
-    read_context: dict[str, Any]
+    read_context: dict[str, SpreadsheetRead]
 
     @classmethod
     def display_type_name(cls) -> str:

cognite/neat/_rules/analysis/__init__.py

@@ -1,4 +1,3 @@
-from ._dms import DMSAnalysis
-from ._information import InformationAnalysis
+from ._base import RulesAnalysis
 
-__all__ = ["DMSAnalysis", "InformationAnalysis"]
+__all__ = ["RulesAnalysis"]