cognite-neat 0.96.2__tar.gz → 0.96.3__tar.gz

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.96.2
+Version: 0.96.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_graph/loaders/_base.py

@@ -6,7 +6,7 @@ from typing import ClassVar, Generic, TypeVar
 from cognite.client import CogniteClient
 from cognite.client.data_classes.capabilities import Capability
 
-from cognite.neat._issues import IssueList, NeatIssue, NeatIssueList
+from cognite.neat._issues import IssueList, NeatIssue
 from cognite.neat._issues.errors import AuthorizationError
 from cognite.neat._store import NeatGraphStore
 from cognite.neat._utils.auxiliary import class_html_doc
@@ -94,6 +94,6 @@ class CDFLoader(BaseLoader[T_Output]):
         client: CogniteClient,
         items: list[T_Output],
         dry_run: bool,
-        read_issues: NeatIssueList,
+        read_issues: IssueList,
     ) -> Iterable[UploadResult]:
         raise NotImplementedError

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_graph/loaders/_rdf2asset.py

@@ -19,7 +19,7 @@ from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError
 
 from cognite.neat._graph._tracking.base import Tracker
 from cognite.neat._graph._tracking.log import LogTracker
-from cognite.neat._issues import IssueList, NeatError, NeatIssue, NeatIssueList
+from cognite.neat._issues import IssueList, NeatError, NeatIssue
 from cognite.neat._issues.errors import ResourceCreationError, ResourceNotFoundError
 from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.analysis._asset import AssetAnalysis
@@ -283,7 +283,7 @@ class AssetLoader(CDFLoader[AssetWrite]):
         client: CogniteClient,
         items: list[AssetWrite] | list[RelationshipWrite] | list[LabelDefinitionWrite],
         dry_run: bool,
-        read_issues: NeatIssueList,
+        read_issues: IssueList,
     ) -> Iterable[UploadResult]:
         if isinstance(items[0], AssetWrite) and all(isinstance(item, AssetWrite) for item in items):
             yield from self._upload_assets_to_cdf(client, cast(list[AssetWrite], items), dry_run, read_issues)
@@ -303,7 +303,7 @@ class AssetLoader(CDFLoader[AssetWrite]):
         client: CogniteClient,
         items: list[LabelDefinitionWrite],
         dry_run: bool,
-        read_issues: NeatIssueList,
+        read_issues: IssueList,
     ) -> Iterable[UploadResult]:
         try:
             created = client.labels.create(items)
@@ -324,7 +324,7 @@ class AssetLoader(CDFLoader[AssetWrite]):
         client: CogniteClient,
         items: list[AssetWrite],
         dry_run: bool,
-        read_issues: NeatIssueList,
+        read_issues: IssueList,
     ) -> Iterable[UploadResult]:
         try:
             upserted = client.assets.upsert(items, mode="replace")
@@ -345,7 +345,7 @@ class AssetLoader(CDFLoader[AssetWrite]):
         client: CogniteClient,
         items: list[RelationshipWrite],
         dry_run: bool,
-        read_issues: NeatIssueList,
+        read_issues: IssueList,
     ) -> Iterable[UploadResult]:
         try:
             upserted = client.relationships.upsert(items, mode="replace")

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_graph/loaders/_rdf2dms.py

@@ -293,7 +293,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         client: CogniteClient,
         items: list[dm.InstanceApply],
         dry_run: bool,
-        read_issues: NeatIssueList,
+        read_issues: IssueList,
     ) -> Iterable[UploadResult]:
         try:
             nodes = [item for item in items if isinstance(item, dm.NodeApply)]

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_issues/__init__.py

@@ -1,7 +1,16 @@
 """This is module contains all the Neat Exceptions (Errors) and Warnings as well
 as some helper classes to handle them like NeatIssueList"""
 
-from ._base import DefaultWarning, IssueList, MultiValueError, NeatError, NeatIssue, NeatIssueList, NeatWarning
+from ._base import (
+    DefaultWarning,
+    IssueList,
+    MultiValueError,
+    NeatError,
+    NeatIssue,
+    NeatIssueList,
+    NeatWarning,
+    catch_warnings,
+)
 
 __all__ = [
     "NeatIssue",
@@ -11,4 +20,5 @@ __all__ = [
     "NeatIssueList",
     "IssueList",
     "MultiValueError",
+    "catch_warnings",
 ]

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_issues/_base.py

@@ -2,7 +2,8 @@ import inspect
 import sys
 import warnings
 from abc import ABC
-from collections.abc import Collection, Hashable, Iterable, Sequence
+from collections.abc import Collection, Hashable, Iterable, Iterator, Sequence
+from contextlib import contextmanager
 from dataclasses import dataclass, fields
 from functools import total_ordering
 from pathlib import Path
@@ -371,9 +372,12 @@ T_NeatIssue = TypeVar("T_NeatIssue", bound=NeatIssue)
 class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
     """This is a generic list of NeatIssues."""
 
-    def __init__(self, issues: Sequence[T_NeatIssue] | None = None, title: str | None = None):
+    def __init__(
+        self, issues: Sequence[T_NeatIssue] | None = None, title: str | None = None, action: str | None = None
+    ):
         super().__init__(issues or [])
         self.title = title
+        self.action = action
 
     @property
     def errors(self) -> Self:
@@ -450,3 +454,18 @@ def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[t
     for s in cls_.__subclasses__():
         yield s
         yield from _get_subclasses(s, False)
+
+
+@contextmanager
+def catch_warnings(
+    issues: IssueList | None = None,
+    warning_cls: type[NeatWarning] = DefaultWarning,
+) -> Iterator[None]:
+    """Catch warnings and append them to the issues list."""
+    with warnings.catch_warnings(record=True) as warning_logger:
+        warnings.simplefilter("always")
+        try:
+            yield None
+        finally:
+            if warning_logger and issues is not None:
+                issues.extend([warning_cls.from_warning(warning) for warning in warning_logger])  # type: ignore[misc]

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_rules/transformers/_verification.py

@@ -1,4 +1,3 @@
-import warnings
 from abc import ABC
 from collections.abc import Iterator
 from contextlib import contextmanager
@@ -6,7 +5,7 @@ from typing import Any, Literal
 
 from pydantic import ValidationError
 
-from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning
+from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning, catch_warnings
 from cognite.neat._issues.errors import NeatTypeError
 from cognite.neat._rules._shared import (
     InputRules,
@@ -46,7 +45,7 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
         if isinstance(rules, ReadRules):
             error_args = rules.read_context
         verified_rules: T_VerifiedRules | None = None
-        with _handle_issues(issues, NeatError, NeatWarning, error_args) as future:
+        with _catch_issues(issues, NeatError, NeatWarning, error_args) as future:
             rules_cls = self._get_rules_cls(in_)
             verified_rules = rules_cls.model_validate(in_.dump())  # type: ignore[assignment]
 
@@ -105,7 +104,7 @@ class _FutureResult:
 
 
 @contextmanager
-def _handle_issues(
+def _catch_issues(
     issues: IssueList,
     error_cls: type[NeatError] = NeatError,
     warning_cls: type[NeatWarning] = NeatWarning,
@@ -121,8 +120,7 @@ def _handle_issues(
     Returns:
         FutureResult: A future result object that can be used to check the result of the context manager.
     """
-    with warnings.catch_warnings(record=True) as warning_logger:
-        warnings.simplefilter("always")
+    with catch_warnings(issues, warning_cls):
         future_result = _FutureResult()
         try:
             yield future_result
@@ -137,6 +135,3 @@ def _handle_issues(
             future_result._result = "failure"
         else:
             future_result._result = "success"
-        finally:
-            if warning_logger:
-                issues.extend([warning_cls.from_warning(warning) for warning in warning_logger])  # type: ignore[misc]

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_session/_base.py

@@ -53,6 +53,7 @@ class NeatSession:
             self._state.verified_rules.append(output.rules)
             if isinstance(output.rules, InformationRules):
                 self._state.store.add_rules(output.rules)
+        output.issues.action = "verify"
         self._state.issue_lists.append(output.issues)
         if output.issues:
             print("You can inspect the issues with the .inspect.issues(...) method.")

cognite_neat-0.96.3/cognite/neat/_session/_inspect.py

@@ -0,0 +1,177 @@
+import difflib
+from typing import Literal, overload
+
+import pandas as pd
+
+from cognite.neat._constants import IN_NOTEBOOK
+from cognite.neat._issues import IssueList
+from cognite.neat._utils.upload import UploadResult, UploadResultCore, UploadResultList
+
+from ._state import SessionState
+from .exceptions import intercept_session_exceptions
+
+
+@intercept_session_exceptions
+class InspectAPI:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+        self.issues = InspectIssues(state)
+        self.outcome = InspectOutcome(state)
+
+    @property
+    def properties(self) -> pd.DataFrame:
+        """Returns the properties of the current data model."""
+        return self._state.last_verified_rule.properties.to_pandas()
+
+
+@intercept_session_exceptions
+class InspectIssues:
+    """Inspect issues of the current data model."""
+
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+
+    @overload
+    def __call__(
+        self,
+        search: str | None = None,
+        return_dataframe: Literal[True] = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
+    ) -> pd.DataFrame: ...
+
+    @overload
+    def __call__(
+        self,
+        search: str | None = None,
+        return_dataframe: Literal[False] = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
+    ) -> None: ...
+
+    def __call__(
+        self,
+        search: str | None = None,
+        return_dataframe: bool = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
+    ) -> pd.DataFrame | None:
+        """Returns the issues of the current data model."""
+        issues = self._state.last_issues
+        if not issues:
+            self._print("No issues found.")
+
+        if issues and search is not None:
+            unique_types = {type(issue).__name__ for issue in issues}
+            closest_match = set(difflib.get_close_matches(search, unique_types))
+            issues = IssueList([issue for issue in issues if type(issue).__name__ in closest_match])
+
+        if IN_NOTEBOOK:
+            from IPython.display import Markdown, display
+
+            issue_str = "\n".join(
+                [f"  * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
+            )
+            message = f"### {len(issues)} issues found\n\n{issue_str}"
+            display(Markdown(message))
+
+        if return_dataframe:
+            return issues.to_pandas()
+        return None
+
+    def _print(self, message: str) -> None:
+        if IN_NOTEBOOK:
+            from IPython.display import Markdown, display
+
+            display(Markdown(message))
+        else:
+            print(message)
+
+    def __repr__(self) -> str:
+        return self.__repr_html__()
+
+    def __repr_html__(self) -> str:
+        return (
+            "Inspect issues by calling .inspect.issues() or "
+            "search for specific issues by calling .inspect.issues('MyTypeWarning')."
+        )
+
+
+@intercept_session_exceptions
+class InspectOutcome:
+    def __init__(self, state: SessionState) -> None:
+        self._state = state
+
+    @staticmethod
+    def _as_set(value: str | list[str] | None) -> set[str] | None:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            return {value}
+        return set(value)
+
+    @overload
+    def __call__(
+        self,
+        name: str | list[str] | None = None,
+        has_errors: bool = False,
+        has_issues: bool = False,
+        return_dataframe: Literal[False] = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
+    ) -> None: ...
+
+    @overload
+    def __call__(
+        self,
+        name: str | list[str] | None = None,
+        has_errors: bool = False,
+        has_issues: bool = False,
+        return_dataframe: Literal[True] = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
+    ) -> pd.DataFrame: ...
+
+    def __call__(
+        self,
+        name: str | list[str] | None = None,
+        has_errors: bool = False,
+        has_issues: bool = False,
+        return_dataframe: bool = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
+    ) -> pd.DataFrame | None:
+        """Returns the outcome of the last upload."""
+        outcome = self._state.last_outcome
+        name_set = self._as_set(name)
+
+        def outcome_filter(item: UploadResultCore) -> bool:
+            nonlocal name_set
+            if name_set and item.name not in name_set:
+                return False
+            if has_errors and not item.error_messages:
+                return False
+            if has_issues and not item.issues:
+                return False
+            return True
+
+        outcome = UploadResultList([item for item in outcome if outcome_filter(item)])
+        if IN_NOTEBOOK:
+            from IPython.display import Markdown, display
+
+            lines: list[str] = []
+            for item in outcome:
+                lines.append(f"### {item.name}")
+                if unique_errors := set(item.error_messages):
+                    lines.append("#### Errors")
+                    for error in unique_errors:
+                        lines.append(f"  * {error}")
+                if unique_issue_messages := set([issue.as_message() for issue in item.issues]):
+                    lines.append("#### Issues")
+                    for issue in unique_issue_messages:
+                        lines.append(f"  * {issue}")
+                if isinstance(item, UploadResult):
+                    dumped = item.dump(aggregate=False)
+                    for key, value in dumped.items():
+                        if key in ["name", "error_messages", "issues"]:
+                            continue
+                        lines.append(f"#### {key}")
+                        if isinstance(value, list):
+                            for v in value:
+                                lines.append(f"  * {v}")
+                        else:
+                            lines.append(f"  * {value}")
+
+            display(Markdown("\n".join(lines)))
+
+        if return_dataframe:
+            return outcome.to_pandas()
+        return None

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_session/_state.py

@@ -7,6 +7,7 @@ from cognite.neat._rules.models.dms._rules import DMSRules
 from cognite.neat._rules.models.information._rules import InformationRules
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
 from cognite.neat._store import NeatGraphStore
+from cognite.neat._utils.upload import UploadResultList
 
 from .exceptions import NeatSessionError
 
@@ -17,6 +18,7 @@ class SessionState:
     input_rules: list[ReadRules] = field(default_factory=list)
     verified_rules: list[VerifiedRules] = field(default_factory=list)
     issue_lists: list[IssueList] = field(default_factory=list)
+    outcome: list[UploadResultList] = field(default_factory=list)
     _store: NeatGraphStore | None = field(init=False, default=None)
 
     @property
@@ -34,6 +36,14 @@ class SessionState:
             raise NeatSessionError("No input data model available. Try using [bold].read[/bold] to load a data model.")
         return self.input_rules[-1]
 
+    @property
+    def last_outcome(self) -> UploadResultList:
+        if not self.outcome:
+            raise NeatSessionError(
+                "No outcome available. Try using [bold].to.cdf[/bold] to upload a data model/instances."
+            )
+        return self.outcome[-1]
+
     @property
     def information_input_rule(self) -> ReadRules | None:
         if self.input_rules:

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_session/_to.py

@@ -4,8 +4,10 @@ from typing import Any, Literal, overload
 from cognite.client import CogniteClient
 
 from cognite.neat._graph import loaders
+from cognite.neat._issues import IssueList, catch_warnings
 from cognite.neat._rules import exporters
 from cognite.neat._session._wizard import space_wizard
+from cognite.neat._utils.upload import UploadResultCore
 
 from ._state import SessionState
 from .exceptions import intercept_session_exceptions
@@ -61,11 +63,19 @@ class CDFToAPI:
 
         return loader.load_into_cdf(self._client)
 
-    def data_model(self, existing_handling: Literal["fail", "skip", "update", "force"] = "skip"):
+    def data_model(self, existing_handling: Literal["fail", "skip", "update", "force"] = "skip", dry_run: bool = False):
         """Export the verified DMS data model to CDF.
 
         Args:
             existing_handling: How to handle if component of data model exists. Defaults to "skip".
+            dry_run: If True, no changes will be made to CDF. Defaults to False.
+
+        ... note::
+
+        - "fail": If any component already exists, the export will fail.
+        - "skip": If any component already exists, it will be skipped.
+        - "update": If any component already exists, it will be updated.
+        - "force": If any component already exists, it will be deleted and recreated.
 
         """
         if not self._state.last_verified_dms_rules:
@@ -76,4 +86,10 @@ class CDFToAPI:
         if not self._client:
             raise ValueError("No client provided!")
 
-        return exporter.export_to_cdf(self._state.last_verified_dms_rules, self._client)
+        conversion_issues = IssueList(action="to.cdf.data_model")
+        with catch_warnings(conversion_issues):
+            result = exporter.export_to_cdf(self._state.last_verified_dms_rules, self._client, dry_run)
+        result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
+        self._state.outcome.append(result)
+        print("You can inspect the details with the .inspect.outcome(...) method.")
+        return result

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/cognite/neat/_utils/upload.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
 from functools import total_ordering
 from typing import Any, Generic
 
-from cognite.neat._issues import NeatIssueList
+from cognite.neat._issues import IssueList
 from cognite.neat._shared import T_ID, NeatList, NeatObject
 
 
@@ -12,7 +12,7 @@ from cognite.neat._shared import T_ID, NeatList, NeatObject
 class UploadResultCore(NeatObject, ABC):
     name: str
     error_messages: list[str] = field(default_factory=list)
-    issues: NeatIssueList = field(default_factory=NeatIssueList)
+    issues: IssueList = field(default_factory=IssueList)
 
     def __lt__(self, other: object) -> bool:
         if isinstance(other, UploadResultCore):
@@ -27,10 +27,19 @@ class UploadResultCore(NeatObject, ABC):
             return NotImplemented
 
     def dump(self, aggregate: bool = True) -> dict[str, Any]:
-        return {"name": self.name}
+        output: dict[str, Any] = {"name": self.name}
+        if self.error_messages:
+            output["error_messages"] = len(self.error_messages) if aggregate else self.error_messages
+        if self.issues:
+            output["issues"] = len(self.issues) if aggregate else [issue.dump() for issue in self.issues]
+        return output
 
 
-class UploadResultList(NeatList[UploadResultCore]): ...
+class UploadResultList(NeatList[UploadResultCore]):
+    def _repr_html_(self) -> str:
+        df = self.to_pandas().fillna(0)
+        df = df.style.format({column: "{:,.0f}".format for column in df.select_dtypes(include="number").columns})
+        return df._repr_html_()  # type: ignore[attr-defined]
 
 
 @dataclass
@@ -85,10 +94,12 @@ class UploadResult(UploadResultCore, Generic[T_ID]):
             output["failed_changed"] = len(self.failed_changed) if aggregate else list(self.failed_changed)
         if self.failed_deleted:
             output["failed_deleted"] = len(self.failed_deleted) if aggregate else list(self.failed_deleted)
-        if self.error_messages:
-            output["error_messages"] = len(self.error_messages) if aggregate else self.error_messages
-        if self.issues:
-            output["issues"] = len(self.issues) if aggregate else [issue.dump() for issue in self.issues]
+        if "error_messages" in output:
+            # Trick to move error_messages to the end of the dict
+            output["error_messages"] = output.pop("error_messages")
+        if "issues" in output:
+            # Trick to move issues to the end of the dict
+            output["issues"] = output.pop("issues")
         return output
 
     def __str__(self) -> str:

cognite_neat-0.96.3/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.96.3"

{cognite_neat-0.96.2 → cognite_neat-0.96.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.96.2"
+version = "0.96.3"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.96.2/cognite/neat/_session/_inspect.py

@@ -1,89 +0,0 @@
-import difflib
-from typing import Literal, overload
-
-import pandas as pd
-
-from cognite.neat._constants import IN_NOTEBOOK
-from cognite.neat._issues import IssueList
-
-from ._state import SessionState
-from .exceptions import intercept_session_exceptions
-
-
-@intercept_session_exceptions
-class InspectAPI:
-    def __init__(self, state: SessionState) -> None:
-        self._state = state
-        self.issues = InspectIssues(state)
-
-    @property
-    def properties(self) -> pd.DataFrame:
-        """Returns the properties of the current data model."""
-        return self._state.last_verified_rule.properties.to_pandas()
-
-
-@intercept_session_exceptions
-class InspectIssues:
-    """Inspect issues of the current data model."""
-
-    def __init__(self, state: SessionState) -> None:
-        self._state = state
-
-    @overload
-    def __call__(
-        self,
-        search: str | None = None,
-        return_dataframe: Literal[True] = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
-    ) -> pd.DataFrame: ...
-
-    @overload
-    def __call__(
-        self,
-        search: str | None = None,
-        return_dataframe: Literal[False] = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
-    ) -> None: ...
-
-    def __call__(
-        self,
-        search: str | None = None,
-        return_dataframe: bool = (False if IN_NOTEBOOK else True),  # type: ignore[assignment]
-    ) -> pd.DataFrame | None:
-        """Returns the issues of the current data model."""
-        issues = self._state.last_issues
-        if not issues:
-            self._print("No issues found.")
-
-        if issues and search is not None:
-            unique_types = {type(issue).__name__ for issue in issues}
-            closest_match = set(difflib.get_close_matches(search, unique_types))
-            issues = IssueList([issue for issue in issues if type(issue).__name__ in closest_match])
-
-        if IN_NOTEBOOK:
-            from IPython.display import Markdown, display
-
-            issue_str = "\n".join(
-                [f"  * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
-            )
-            message = f"### {len(issues)} issues found\n\n{issue_str}"
-            display(Markdown(message))
-
-        if return_dataframe:
-            return issues.to_pandas()
-        return None
-
-    def _print(self, message: str) -> None:
-        if IN_NOTEBOOK:
-            from IPython.display import Markdown, display
-
-            display(Markdown(message))
-        else:
-            print(message)
-
-    def __repr__(self) -> str:
-        return self.__repr_html__()
-
-    def __repr_html__(self) -> str:
-        return (
-            "Inspect issues by calling .inspect.issues() or "
-            "search for specific issues by calling .inspect.issues('MyTypeWarning')."
-        )

cognite_neat-0.96.2/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.96.2"