cognite-neat 0.78.3__py3-none-any.whl → 0.78.5__py3-none-any.whl

cognite/neat/rules/issues/base.py

@@ -1,87 +1,43 @@
-import sys
-import warnings
-from abc import ABC, abstractmethod
-from collections import UserList
-from collections.abc import Sequence
+from abc import ABC
 from dataclasses import dataclass
-from functools import total_ordering
-from typing import Any, ClassVar
-from warnings import WarningMessage
+from typing import Any
 
-import pandas as pd
 from pydantic_core import ErrorDetails
 
-if sys.version_info < (3, 11):
-    from exceptiongroup import ExceptionGroup
-else:
-    pass
+from cognite.neat.issues import MultiValueError, NeatError, NeatIssue, NeatIssueList, NeatWarning
 
 __all__ = [
     "ValidationIssue",
     "NeatValidationError",
     "DefaultPydanticError",
     "ValidationWarning",
-    "DefaultWarning",
     "IssueList",
     "MultiValueError",
 ]
 
 
-@total_ordering
 @dataclass(frozen=True)
-class ValidationIssue(ABC):
-    description: ClassVar[str]
-    fix: ClassVar[str]
-
-    def message(self) -> str:
-        """Return a human-readable message for the issue.
-
-        This is the default implementation, which returns the description.
-        It is recommended to override this method in subclasses with a more
-        specific message.
-        """
-        return self.description
-
-    @abstractmethod
-    def dump(self) -> dict[str, Any]:
-        """Return a dictionary representation of the issue."""
-        raise NotImplementedError()
-
-    def __lt__(self, other: "ValidationIssue") -> bool:
-        if not isinstance(other, ValidationIssue):
-            return NotImplemented
-        return (type(self).__name__, self.message()) < (type(other).__name__, other.message())
-
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, ValidationIssue):
-            return NotImplemented
-        return (type(self).__name__, self.message()) == (type(other).__name__, other.message())
+class ValidationIssue(NeatIssue, ABC): ...
 
 
 @dataclass(frozen=True)
-class NeatValidationError(ValidationIssue, ABC):
-    def dump(self) -> dict[str, Any]:
-        return {"error": type(self).__name__}
-
+class NeatValidationError(NeatError, ValidationIssue, ABC):
     @classmethod
     def from_pydantic_errors(cls, errors: list[ErrorDetails], **kwargs) -> "list[NeatValidationError]":
         """Convert a list of pydantic errors to a list of Error instances.
 
         This is intended to be overridden in subclasses to handle specific error types.
         """
-        all_errors = []
+        all_errors: list[NeatValidationError] = []
         for error in errors:
             if isinstance(ctx := error.get("ctx"), dict) and isinstance(
                 multi_error := ctx.get("error"), MultiValueError
             ):
-                all_errors.extend(multi_error.errors)
+                all_errors.extend(multi_error.errors)  # type: ignore[arg-type]
             else:
                 all_errors.append(DefaultPydanticError.from_pydantic_error(error))
         return all_errors
 
-    def as_exception(self) -> Exception:
-        return ValueError(self.message())
-
 
 @dataclass(frozen=True)
 class DefaultPydanticError(NeatValidationError):
@@ -120,87 +76,7 @@ class DefaultPydanticError(NeatValidationError):
 
 
 @dataclass(frozen=True)
-class ValidationWarning(ValidationIssue, ABC, UserWarning):
-    def dump(self) -> dict[str, Any]:
-        return {"warning": type(self).__name__}
-
-    @classmethod
-    def from_warning(cls, warning: WarningMessage) -> "ValidationWarning":
-        return DefaultWarning.from_warning_message(warning)
-
-
-@dataclass(frozen=True)
-class DefaultWarning(ValidationWarning):
-    description = "A warning was raised during validation."
-    fix = "No fix is available."
-
-    warning: str | Warning
-    category: type[Warning]
-    source: str | None = None
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["msg"] = str(self.warning)
-        output["category"] = self.category.__name__
-        output["source"] = self.source
-        return output
-
-    @classmethod
-    def from_warning_message(cls, warning: WarningMessage) -> "ValidationWarning":
-        if isinstance(warning.message, ValidationWarning):
-            return warning.message
-
-        return cls(
-            warning=warning.message,
-            category=warning.category,
-            source=warning.source,
-        )
-
-    def message(self) -> str:
-        return str(self.warning)
-
-
-class IssueList(UserList[ValidationIssue]):
-    def __init__(self, issues: Sequence[ValidationIssue] | None = None, title: str | None = None):
-        super().__init__(issues or [])
-        self.title = title
-
-    @property
-    def errors(self) -> "IssueList":
-        return IssueList([issue for issue in self if isinstance(issue, NeatValidationError)])
-
-    @property
-    def has_errors(self) -> bool:
-        return any(isinstance(issue, NeatValidationError) for issue in self)
-
-    @property
-    def warnings(self) -> "IssueList":
-        return IssueList([issue for issue in self if isinstance(issue, ValidationWarning)])
-
-    def as_errors(self) -> ExceptionGroup:
-        return ExceptionGroup(
-            "Validation failed",
-            [ValueError(issue.message()) for issue in self if isinstance(issue, NeatValidationError)],
-        )
-
-    def trigger_warnings(self) -> None:
-        for warning in [issue for issue in self if isinstance(issue, ValidationWarning)]:
-            warnings.warn(warning, stacklevel=2)
-
-    def to_pandas(self) -> pd.DataFrame:
-        return pd.DataFrame([issue.dump() for issue in self])
-
-    def _repr_html_(self) -> str | None:
-        return self.to_pandas()._repr_html_()  # type: ignore[operator]
-
-
-class MultiValueError(ValueError):
-    """This is a container for multiple errors.
-
-    It is used in the pydantic field_validator/model_validator to collect multiple errors, which
-    can then be caught in a try-except block and returned as an IssueList.
+class ValidationWarning(NeatWarning, ValidationIssue, ABC): ...
 
-    """
 
-    def __init__(self, errors: Sequence[NeatValidationError]):
-        self.errors = list(errors)
+class IssueList(NeatIssueList[ValidationIssue]): ...

cognite/neat/rules/issues/spreadsheet.py

@@ -8,9 +8,10 @@ from cognite.client.data_classes import data_modeling as dm
 from cognite.client.data_classes.data_modeling import ContainerId, ViewId
 from pydantic_core import ErrorDetails
 
+from cognite.neat.issues import MultiValueError
 from cognite.neat.utils.spreadsheet import SpreadsheetRead
 
-from .base import DefaultPydanticError, MultiValueError, NeatValidationError
+from .base import DefaultPydanticError, NeatValidationError
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -69,7 +70,7 @@ class InvalidSheetError(NeatValidationError, ABC):
                             new_row = reader.adjusted_row_number(caught_error.row)
                             # The error is frozen, so we have to use __setattr__ to change the row number
                             object.__setattr__(caught_error, "row", new_row)
-                        output.append(caught_error)
+                        output.append(caught_error)  # type: ignore[arg-type]
                     continue
 
             if len(error["loc"]) >= 4:

cognite/neat/rules/models/_base.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 import math
 import sys
 import types
+from abc import abstractmethod
 from collections.abc import Callable, Iterator
 from functools import wraps
 from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
@@ -245,6 +246,11 @@ class BaseMetadata(RuleModel):
     def include_role(self, serializer: Callable) -> dict:
         return {"role": self.role.value, **serializer(self)}
 
+    @abstractmethod
+    def as_identifier(self) -> str:
+        """Returns a unique identifier for the metadata."""
+        raise NotImplementedError()
+
 
 class BaseRules(RuleModel):
     """

cognite/neat/rules/models/dms/_rules.py

@@ -132,6 +132,9 @@ class DMSMetadata(BaseMetadata):
             views=[],
         )
 
+    def as_identifier(self) -> str:
+        return repr(self.as_data_model_id())
+
     @classmethod
     def _get_description_and_creator(cls, description_raw: str | None) -> tuple[str | None, list[str]]:
         if description_raw and (description_match := re.search(r"Creator: (.+)", description_raw)):

cognite/neat/rules/models/dms/_schema.py

@@ -2,17 +2,27 @@ import json
 import sys
 import warnings
 import zipfile
-from collections import Counter, defaultdict
+from collections import ChainMap, Counter, defaultdict
+from collections.abc import Iterable, MutableMapping
 from dataclasses import Field, dataclass, field, fields
 from pathlib import Path
-from typing import Any, ClassVar, cast
+from typing import Any, ClassVar, Literal, cast
 
 import yaml
 from cognite.client import CogniteClient
 from cognite.client import data_modeling as dm
 from cognite.client.data_classes import DatabaseWrite, DatabaseWriteList, TransformationWrite, TransformationWriteList
 from cognite.client.data_classes.data_modeling import ViewApply
-from cognite.client.data_classes.data_modeling.views import ReverseDirectRelation
+from cognite.client.data_classes.data_modeling.views import (
+    ReverseDirectRelation,
+    ReverseDirectRelationApply,
+    SingleEdgeConnection,
+    SingleEdgeConnectionApply,
+    SingleReverseDirectRelation,
+    SingleReverseDirectRelationApply,
+    ViewProperty,
+    ViewPropertyApply,
+)
 from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
 
 from cognite.neat.rules import issues
@@ -668,6 +678,126 @@ class DMSSchema:
         referenced_spaces |= {s.space for s in self.spaces.values()}
         return referenced_spaces
 
+    def as_read_model(self) -> dm.DataModel[dm.View]:
+        if self.data_model is None:
+            raise ValueError("Data model is not defined")
+        all_containers = self.containers.copy()
+        all_views = self.views.copy()
+        for other_schema in [self.reference, self.last]:
+            if other_schema:
+                all_containers |= other_schema.containers
+                all_views |= other_schema.views
+
+        views: list[dm.View] = []
+        for view in self.views.values():
+            referenced_containers = ContainerApplyDict()
+            properties: dict[str, ViewProperty] = {}
+            # ChainMap is used to merge properties from the view and its parents
+            # Note that the order of the ChainMap is important, as the first dictionary has the highest priority
+            # So if a child and parent have the same property, the child property will be used.
+            write_properties = ChainMap(view.properties, *(all_views[v].properties for v in view.implements or []))  # type: ignore[arg-type]
+            for prop_name, prop in write_properties.items():
+                read_prop = self._as_read_properties(prop, all_containers)
+                if isinstance(read_prop, dm.MappedProperty) and read_prop.container not in referenced_containers:
+                    referenced_containers[read_prop.container] = all_containers[read_prop.container]
+                properties[prop_name] = read_prop
+
+            read_view = dm.View(
+                space=view.space,
+                external_id=view.external_id,
+                version=view.version,
+                description=view.description,
+                name=view.name,
+                filter=view.filter,
+                implements=view.implements.copy(),
+                used_for=self._used_for(referenced_containers.values()),
+                writable=self._writable(properties.values(), referenced_containers.values()),
+                properties=properties,
+                is_global=False,
+                last_updated_time=0,
+                created_time=0,
+            )
+            views.append(read_view)
+
+        return dm.DataModel(
+            space=self.data_model.space,
+            external_id=self.data_model.external_id,
+            version=self.data_model.version,
+            name=self.data_model.name,
+            description=self.data_model.description,
+            views=views,
+            is_global=False,
+            last_updated_time=0,
+            created_time=0,
+        )
+
+    @staticmethod
+    def _as_read_properties(
+        write: ViewPropertyApply, all_containers: MutableMapping[dm.ContainerId, dm.ContainerApply]
+    ) -> ViewProperty:
+        if isinstance(write, dm.MappedPropertyApply):
+            container_prop = all_containers[write.container].properties[write.container_property_identifier]
+            return dm.MappedProperty(
+                container=write.container,
+                container_property_identifier=write.container_property_identifier,
+                name=write.name,
+                description=write.description,
+                source=write.source,
+                type=container_prop.type,
+                nullable=container_prop.nullable,
+                auto_increment=container_prop.auto_increment,
+                # Likely bug in SDK.
+                default_value=container_prop.default_value,  # type: ignore[arg-type]
+            )
+        if isinstance(write, dm.EdgeConnectionApply):
+            edge_cls = SingleEdgeConnection if isinstance(write, SingleEdgeConnectionApply) else dm.MultiEdgeConnection
+            return edge_cls(
+                type=write.type,
+                source=write.source,
+                name=write.name,
+                description=write.description,
+                edge_source=write.edge_source,
+                direction=write.direction,
+            )
+        if isinstance(write, ReverseDirectRelationApply):
+            relation_cls = (
+                SingleReverseDirectRelation
+                if isinstance(write, SingleReverseDirectRelationApply)
+                else dm.MultiReverseDirectRelation
+            )
+            return relation_cls(
+                source=write.source,
+                through=write.through,
+                name=write.name,
+                description=write.description,
+            )
+        raise ValueError(f"Cannot convert {write} to read format")
+
+    @staticmethod
+    def _used_for(containers: Iterable[dm.ContainerApply]) -> Literal["node", "edge", "all"]:
+        used_for = {container.used_for for container in containers}
+        if used_for == {"node"}:
+            return "node"
+        if used_for == {"edge"}:
+            return "edge"
+        return "all"
+
+    @staticmethod
+    def _writable(properties: Iterable[ViewProperty], containers: Iterable[dm.ContainerApply]) -> bool:
+        used_properties = {
+            (prop.container, prop.container_property_identifier)
+            for prop in properties
+            if isinstance(prop, dm.MappedProperty)
+        }
+        required_properties = {
+            (container.as_id(), prop_id)
+            for container in containers
+            for prop_id, prop in container.properties.items()
+            if not prop.nullable
+        }
+        # If a container has a required property that is not used by the view, the view is not writable
+        return not bool(required_properties - used_properties)
+
 
 @dataclass
 class PipelineSchema(DMSSchema):

cognite/neat/rules/models/domain.py

@@ -21,6 +21,9 @@ class DomainMetadata(BaseMetadata):
     role: ClassVar[RoleTypes] = RoleTypes.domain_expert
     creator: StrOrListType
 
+    def as_identifier(self) -> str:
+        return "DomainRules"
+
 
 class DomainProperty(SheetEntity):
     class_: ClassEntity = Field(alias="Class")

cognite/neat/rules/models/information/_rules.py

@@ -8,8 +8,8 @@ from pydantic.main import IncEx
 from rdflib import Namespace
 
 from cognite.neat.constants import PREFIXES
+from cognite.neat.issues import MultiValueError
 from cognite.neat.rules import exceptions, issues
-from cognite.neat.rules.issues.base import MultiValueError
 from cognite.neat.rules.models._base import (
     BaseMetadata,
     BaseRules,
@@ -116,6 +116,9 @@ class InformationMetadata(BaseMetadata):
     def as_enum_model_type(cls, value: str) -> DataModelType:
         return DataModelType(value)
 
+    def as_identifier(self) -> str:
+        return f"{self.prefix}:{self.name}"
+
 
 class InformationClass(SheetEntity):
     """

cognite/neat/{rules/exporters/_models.py → utils/upload.py}

@@ -2,29 +2,31 @@ from abc import ABC
 from dataclasses import dataclass, field
 from functools import total_ordering
 
-from cognite.neat.rules.issues import IssueList
+from cognite.neat.issues import NeatIssueList
 
 
 @total_ordering
 @dataclass
 class UploadResultCore(ABC):
     name: str
+    error_messages: list[str] = field(default_factory=list)
+    issues: NeatIssueList = field(default_factory=NeatIssueList)
 
     def __lt__(self, other: object) -> bool:
-        if isinstance(other, UploadResult):
+        if isinstance(other, UploadDiffsCount):
             return self.name < other.name
         else:
             return NotImplemented
 
     def __eq__(self, other: object) -> bool:
-        if isinstance(other, UploadResult):
+        if isinstance(other, UploadDiffsCount):
             return self.name == other.name
         else:
             return NotImplemented
 
 
 @dataclass
-class UploadResult(UploadResultCore):
+class UploadDiffsCount(UploadResultCore):
     created: int = 0
     deleted: int = 0
     changed: int = 0
@@ -33,8 +35,6 @@ class UploadResult(UploadResultCore):
     failed_created: int = 0
     failed_changed: int = 0
     failed_deleted: int = 0
-    error_messages: list[str] = field(default_factory=list)
-    issues: IssueList = field(default_factory=IssueList)
 
     @property
     def total(self) -> int:
@@ -64,3 +64,23 @@ class UploadResult(UploadResultCore):
             line.append(f"failed to delete {self.failed_deleted}")
 
         return f"{self.name.title()}: {', '.join(line)}"
+
+
+@dataclass
+class UploadResultIDs(UploadResultCore):
+    success: list[str] = field(default_factory=list)
+    failed: list[str] = field(default_factory=list)
+
+
+@dataclass
+class UploadDiffsID(UploadResultCore):
+    created: list[str] = field(default_factory=list)
+    changed: list[str] = field(default_factory=list)
+    unchanged: list[str] = field(default_factory=list)
+    failed: list[str] = field(default_factory=list)
+
+    def as_upload_result_ids(self) -> UploadResultIDs:
+        result = UploadResultIDs(name=self.name, error_messages=self.error_messages, issues=self.issues)
+        result.success = self.created + self.changed + self.unchanged
+        result.failed = self.failed
+        return result

cognite/neat/utils/utils.py

@@ -355,3 +355,27 @@ def get_inheritance_path(child: Any, child_parent: dict[Any, list[Any]]) -> list
 
 def replace_non_alphanumeric_with_underscore(text):
     return re.sub(r"\W+", "_", text)
+
+
+def string_to_ideal_type(input_string: str) -> int | bool | float | datetime | str:
+    try:
+        # Try converting to int
+        return int(input_string)
+    except ValueError:
+        try:
+            # Try converting to float
+            return float(input_string)  # type: ignore
+        except ValueError:
+            if input_string.lower() == "true":
+                # Return True if input is 'true'
+                return True
+            elif input_string.lower() == "false":
+                # Return False if input is 'false'
+                return False
+            else:
+                try:
+                    # Try converting to datetime
+                    return datetime.fromisoformat(input_string)  # type: ignore
+                except ValueError:
+                    # Return the input string if no conversion is possible
+                    return input_string

{cognite_neat-0.78.3.dist-info → cognite_neat-0.78.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.78.3
+Version: 0.78.5
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -22,7 +22,7 @@ Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
 Requires-Dist: cognite-sdk (>=7.37.0,<8.0.0)
 Requires-Dist: deepdiff
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
-Requires-Dist: fastapi (>=0.110,<0.111)
+Requires-Dist: fastapi (>=0,<1)
 Requires-Dist: google-api-python-client ; extra == "google" or extra == "all"
 Requires-Dist: google-auth-oauthlib ; extra == "google" or extra == "all"
 Requires-Dist: gspread ; extra == "google" or extra == "all"