cognite-neat 0.104.0__py3-none-any.whl → 0.105.1__py3-none-any.whl

cognite/neat/_issues/_base.py

@@ -425,6 +425,10 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
         """Return True if this list contains any errors of the given type."""
         return any(isinstance(issue, error_type) for issue in self)
 
+    def has_warning_type(self, warning_type: type[NeatWarning]) -> bool:
+        """Return True if this list contains any warnings of the given type."""
+        return any(isinstance(issue, warning_type) for issue in self)
+
     def as_errors(self, operation: str = "Operation failed") -> ExceptionGroup:
         """Return an ExceptionGroup with all the errors in this list."""
         return ExceptionGroup(
@@ -505,55 +509,35 @@ def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[t
 
 
 @contextmanager
-def catch_warnings(
-    issues: IssueList | None = None,
-    warning_cls: type[NeatWarning] = DefaultWarning,
-) -> Iterator[None]:
+def catch_warnings() -> Iterator[IssueList]:
     """Catch warnings and append them to the issues list."""
+    issues = IssueList()
     with warnings.catch_warnings(record=True) as warning_logger:
         warnings.simplefilter("always")
         try:
-            yield None
+            yield issues
         finally:
-            if warning_logger and issues is not None:
-                issues.extend([warning_cls.from_warning(warning) for warning in warning_logger])  # type: ignore[misc]
-
-
-class FutureResult:
-    def __init__(self) -> None:
-        self._result: Literal["success", "failure", "pending"] = "pending"
-
-    @property
-    def result(self) -> Literal["success", "failure", "pending"]:
-        return self._result
+            if warning_logger:
+                issues.extend([NeatWarning.from_warning(warning) for warning in warning_logger])  # type: ignore[misc]
 
 
 @contextmanager
-def catch_issues(
-    issues: IssueList,
-    error_cls: type[NeatError] = NeatError,
-    warning_cls: type[NeatWarning] = NeatWarning,
-    error_args: dict[str, Any] | None = None,
-) -> Iterator[FutureResult]:
+def catch_issues(error_args: dict[str, Any] | None = None) -> Iterator[IssueList]:
     """This is an internal help function to handle issues and warnings.
 
     Args:
-        issues: The issues list to append to.
-        error_cls: The class used to convert errors to issues.
-        warning_cls:  The class used to convert warnings to issues.
+        error_args: Additional arguments to pass to the error class. The only use case as of (2025-01-03) is to pass
+            the read_info_by_sheet to the error class such that the row numbers can be adjusted to match the source
+            spreadsheet.
 
     Returns:
-        FutureResult: A future result object that can be used to check the result of the context manager.
+        IssueList: The list of issues.
+
     """
-    with catch_warnings(issues, warning_cls):
-        future_result = FutureResult()
+    with catch_warnings() as issues:
         try:
-            yield future_result
+            yield issues
         except ValidationError as e:
-            issues.extend(error_cls.from_errors(e.errors(), **(error_args or {})))  # type: ignore[arg-type]
-            future_result._result = "failure"
+            issues.extend(NeatError.from_errors(e.errors(), **(error_args or {})))  # type: ignore[arg-type]
         except (NeatError, MultiValueError) as e:
-            issues.extend(error_cls.from_errors([e], **(error_args or {})))  # type: ignore[arg-type, list-item]
-            future_result._result = "failure"
-        else:
-            future_result._result = "success"
+            issues.extend(NeatError.from_errors([e], **(error_args or {})))  # type: ignore[arg-type, list-item]

cognite/neat/_issues/warnings/__init__.py

@@ -13,7 +13,7 @@ from ._external import (
     FileReadWarning,
     FileTypeUnexpectedWarning,
 )
-from ._general import NeatValueWarning, NotSupportedWarning, RegexViolationWarning
+from ._general import MissingCogniteClientWarning, NeatValueWarning, NotSupportedWarning, RegexViolationWarning
 from ._models import (
     BreakingModelingPrincipleWarning,
     CDFNotSupportedWarning,
@@ -28,6 +28,7 @@ from ._models import (
 from ._properties import (
     PropertyDataTypeConversionWarning,
     PropertyDefinitionDuplicatedWarning,
+    PropertyDirectRelationLimitWarning,
     PropertyNotFoundWarning,
     PropertyOverwritingWarning,
     PropertySkippedWarning,
@@ -41,6 +42,7 @@ from ._resources import (
     ResourceRetrievalWarning,
     ResourcesDuplicatedWarning,
     ResourceTypeNotSupportedWarning,
+    ResourceUnknownWarning,
 )
 
 __all__ = [
@@ -53,6 +55,7 @@ __all__ = [
     "FileMissingRequiredFieldWarning",
     "FileReadWarning",
     "FileTypeUnexpectedWarning",
+    "MissingCogniteClientWarning",
     "NeatValueWarning",
     "NotSupportedHasDataFilterLimitWarning",
     "NotSupportedViewContainerLimitWarning",
@@ -62,6 +65,7 @@ __all__ = [
     "PrincipleSolutionBuildsOnEnterpriseWarning",
     "PropertyDataTypeConversionWarning",
     "PropertyDefinitionDuplicatedWarning",
+    "PropertyDirectRelationLimitWarning",
     "PropertyNotFoundWarning",
     "PropertyOverwritingWarning",
     "PropertySkippedWarning",
@@ -73,6 +77,7 @@ __all__ = [
     "ResourceRegexViolationWarning",
     "ResourceRetrievalWarning",
     "ResourceTypeNotSupportedWarning",
+    "ResourceUnknownWarning",
     "ResourcesDuplicatedWarning",
     "UndefinedViewWarning",
     "UserModelingWarning",

cognite/neat/_issues/warnings/_general.py

@@ -27,3 +27,10 @@ class RegexViolationWarning(NeatWarning):
     identifier: str
     pattern_name: str
     motivation: str | None = None
+
+
+@dataclass(unsafe_hash=True)
+class MissingCogniteClientWarning(NeatWarning):
+    """Missing Cognite Client required for {functionality}"""
+
+    functionality: str

cognite/neat/_issues/warnings/_properties.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Generic
 
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT
 from cognite.neat._issues._base import ResourceType
 
 from ._resources import ResourceNeatWarning, T_Identifier, T_ReferenceIdentifier
@@ -77,3 +78,13 @@ class PropertyDataTypeConversionWarning(PropertyWarning[T_Identifier]):
     """The {resource_type} with identifier {identifier} failed to convert the property {property_name}: {error}"""
 
     error: str
+
+
+@dataclass(unsafe_hash=True)
+class PropertyDirectRelationLimitWarning(PropertyWarning[T_Identifier]):
+    """The listable direct relation property {property_name} in the {resource_type} with identifier {identifier}
+    has more than {limit} relations. The relations will be sorted by (space, externalId) and truncated."""
+
+    resource_type = "view"
+
+    limit: int = DMS_DIRECT_RELATION_LIST_LIMIT

cognite/neat/_issues/warnings/_resources.py

@@ -39,6 +39,17 @@ class ResourceNotFoundWarning(ResourceNeatWarning, Generic[T_Identifier, T_Refer
     referred_type: str
 
 
+@dataclass(unsafe_hash=True)
+class ResourceUnknownWarning(ResourceNeatWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier} referred by {referred_type} {referred_by} is unknown.
+    Will continue, but the model is incomplete."""
+
+    referred_by: T_ReferenceIdentifier
+    referred_type: str
+
+    fix = "You can maybe retrieve the resource from the CDF."
+
+
 @dataclass(unsafe_hash=True)
 class ResourceNotDefinedWarning(ResourceNeatWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
     """The {resource_type} {identifier} is not defined in the {location}"""

cognite/neat/_rules/exporters/_rules2dms.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Generic, Literal
 
+from cognite.client import data_modeling as dm
 from cognite.client.data_classes._base import (
     T_CogniteResourceList,
     T_WritableCogniteResource,
@@ -19,7 +20,7 @@ from cognite.client.exceptions import CogniteAPIError
 
 from cognite.neat._client import DataModelingLoader, NeatClient
 from cognite.neat._client._api.data_modeling_loaders import MultiCogniteAPIError, T_WritableCogniteResourceList
-from cognite.neat._client.data_classes.data_modeling import Component
+from cognite.neat._client.data_classes.data_modeling import Component, ViewApplyDict
 from cognite.neat._client.data_classes.schema import DMSSchema
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.warnings import (
@@ -199,6 +200,10 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
     ) -> Iterable[UploadResult]:
         schema = self.export(rules)
 
+        # The CDF UI does not deal well with a child view overwriting a parent property with the same name
+        # This is a workaround to remove the duplicated properties
+        self._remove_duplicated_properties(schema.views, client)
+
         categorized_items_by_loader = self._categorize_by_loader(client, schema)
 
         is_failing = self.existing == "fail" and any(
@@ -376,3 +381,32 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
                 for data_model in data_models
                 if (data_model.space, data_model.external_id) != (space, external_id)
             ]
+
+    @staticmethod
+    def _remove_duplicated_properties(views: ViewApplyDict, client: NeatClient) -> None:
+        parent_view_ids = {parent for view in views.values() for parent in view.implements}
+        parent_view_list = client.data_modeling.views.retrieve(
+            list(parent_view_ids), include_inherited_properties=False
+        )
+        parent_view_by_id = {view.as_id(): view.as_write() for view in parent_view_list}
+        for view in views.values():
+            if view.implements is None:
+                continue
+            for parent_id in view.implements:
+                if not (parent_view := parent_view_by_id.get(parent_id)):
+                    continue
+                for shared_prop_id in set(view.properties or {}) & set(parent_view.properties or {}):
+                    if view.properties is None or parent_view.properties is None:
+                        continue
+                    prop = view.properties[shared_prop_id]
+                    parent_prop = parent_view.properties[shared_prop_id]
+                    if (
+                        isinstance(prop, dm.MappedPropertyApply)
+                        and isinstance(parent_prop, dm.MappedPropertyApply)
+                        and (
+                            prop.container_property_identifier == parent_prop.container_property_identifier
+                            and prop.container == parent_prop.container
+                            and prop.source == parent_prop.source
+                        )
+                    ):
+                        view.properties.pop(shared_prop_id)

cognite/neat/_rules/exporters/_rules2excel.py

@@ -229,8 +229,8 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
                 if isinstance(selected_column, MergedCell):
                     selected_column = column_cells[1]
 
-                current = sheet.column_dimensions[selected_column.column_letter].width or (max_length + 0.5)
-                sheet.column_dimensions[selected_column.column_letter].width = min(
+                current = sheet.column_dimensions[selected_column.column_letter].width or (max_length + 0.5)  # type: ignore[union-attr]
+                sheet.column_dimensions[selected_column.column_letter].width = min(  # type: ignore[union-attr]
                     max(current, max_length + 0.5), MAX_COLUMN_WIDTH
                 )
         return None

cognite/neat/_rules/importers/_dms2rules.py

@@ -1,6 +1,6 @@
-from collections import Counter, defaultdict
+from collections import defaultdict
 from collections.abc import Collection, Iterable, Sequence
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 from typing import Literal, cast
 
@@ -20,12 +20,19 @@ from cognite.client.utils import ms_to_datetime
 
 from cognite.neat._client import NeatClient
 from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
-from cognite.neat._issues.errors import FileTypeUnexpectedError, ResourceMissingIdentifierError, ResourceRetrievalError
+from cognite.neat._issues.errors import (
+    FileTypeUnexpectedError,
+    NeatValueError,
+    ResourceMissingIdentifierError,
+    ResourceRetrievalError,
+)
 from cognite.neat._issues.warnings import (
+    MissingCogniteClientWarning,
     PropertyNotFoundWarning,
     PropertyTypeNotSupportedWarning,
     ResourceNotFoundWarning,
     ResourcesDuplicatedWarning,
+    ResourceUnknownWarning,
 )
 from cognite.neat._rules._shared import ReadRules
 from cognite.neat._rules.importers._base import BaseImporter, _handle_issues
@@ -60,7 +67,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         schema: The schema containing the data model.
         read_issues: A list of issues that occurred during the import.
         metadata: Metadata for the data model.
-        ref_metadata: Metadata for the reference data model.
 
     """
 
@@ -69,30 +75,23 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         schema: DMSSchema,
         read_issues: Sequence[NeatIssue] | None = None,
         metadata: DMSInputMetadata | None = None,
-        ref_metadata: DMSInputMetadata | None = None,
+        referenced_containers: Iterable[dm.ContainerApply] | None = None,
     ):
-        # Calling this root schema to distinguish it from
-        # * User Schema
-        # * Reference Schema
-        self.root_schema = schema
+        self.schema = schema
         self.metadata = metadata
-        self.ref_metadata = ref_metadata
         self.issue_list = IssueList(read_issues)
         self._all_containers_by_id = schema.containers.copy()
         self._all_views_by_id = schema.views.copy()
-
-    def update_referenced_containers(self, containers: Iterable[dm.ContainerApply]) -> None:
-        """Update the referenced containers. This is useful to add Cognite containers identified after the root schema
-        is read"""
-        for container in containers:
-            if container.as_id() in self._all_containers_by_id:
-                continue
-            self._all_containers_by_id[container.as_id()] = container
+        if referenced_containers is not None:
+            for container in referenced_containers:
+                if container.as_id() in self._all_containers_by_id:
+                    continue
+                self._all_containers_by_id[container.as_id()] = container
 
     @property
     def description(self) -> str:
-        if self.root_schema.data_model is not None:
-            identifier = f"{self.root_schema.data_model.as_id().as_tuple()!s}"
+        if self.schema.data_model is not None:
+            identifier = f"{self.schema.data_model.as_id().as_tuple()!s}"
         else:
             identifier = "Unknown"
         return f"DMS Data model {identifier} read as unverified data model"
@@ -107,8 +106,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
 
         Args:
             client: Instantiated CogniteClient to retrieve data model.
-            reference_model_id: The reference data model to retrieve. This is the data model that
-                the given data model is built on top of, typically, an enterprise data model.
             data_model_id: Data Model to retrieve.
 
         Returns:
@@ -141,7 +138,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
 
         metadata = cls._create_metadata_from_model(user_model)
 
-        return cls(schema, issue_list, metadata, None)
+        return cls(
+            schema,
+            issue_list,
+            metadata,
+            referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client),
+        )
 
     @classmethod
     def _find_model_in_list(
@@ -182,15 +184,17 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         )
 
     @classmethod
-    def from_directory(cls, directory: str | Path) -> "DMSImporter":
+    def from_directory(cls, directory: str | Path, client: NeatClient | None = None) -> "DMSImporter":
         issue_list = IssueList()
         with _handle_issues(issue_list) as _:
             schema = DMSSchema.from_directory(directory)
         # If there were errors during the import, the to_rules
-        return cls(schema, issue_list)
+        return cls(
+            schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
+        )
 
     @classmethod
-    def from_zip_file(cls, zip_file: str | Path) -> "DMSImporter":
+    def from_zip_file(cls, zip_file: str | Path, client: NeatClient | None = None) -> "DMSImporter":
         if Path(zip_file).suffix != ".zip":
             return cls(
                 DMSSchema(),
@@ -199,30 +203,49 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         issue_list = IssueList()
         with _handle_issues(issue_list) as _:
             schema = DMSSchema.from_zip(zip_file)
-        return cls(schema, issue_list)
+        return cls(
+            schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
+        )
+
+    @classmethod
+    def _lookup_referenced_containers(
+        cls, schema: DMSSchema, issue_list: IssueList, client: NeatClient | None = None
+    ) -> Iterable[dm.ContainerApply]:
+        ref_containers = schema.externally_referenced_containers()
+        if not ref_containers:
+            return []
+        elif client is None:
+            id_ = ""
+            if schema.data_model:
+                id_ = f" {schema.data_model.as_id()!r}"
+            issue_list.append(MissingCogniteClientWarning(f"importing full DMS model{id_}"))
+            return []
+        return client.loaders.containers.retrieve(list(ref_containers), format="write")
+
+    @classmethod
+    def from_path(cls, path: Path, client: NeatClient | None = None) -> "DMSImporter":
+        if path.is_file():
+            return cls.from_zip_file(path, client)
+        elif path.is_dir():
+            return cls.from_directory(path, client)
+        else:
+            raise NeatValueError(f"Unsupported YAML format: {format}")
 
     def to_rules(self) -> ReadRules[DMSInputRules]:
         if self.issue_list.has_errors:
             # In case there were errors during the import, the to_rules method will return None
-            self._end = datetime.now(timezone.utc)
             self.issue_list.trigger_warnings()
             raise MultiValueError(self.issue_list.errors)
 
-        if not self.root_schema.data_model:
-            self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.root_schema).__name__))
-            self._end = datetime.now(timezone.utc)
+        if not self.schema.data_model:
+            self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.schema).__name__))
             self.issue_list.trigger_warnings()
             raise MultiValueError(self.issue_list.errors)
 
-        model = self.root_schema.data_model
+        model = self.schema.data_model
 
-        user_rules = self._create_rule_components(
-            model,
-            self.root_schema,
-            self.metadata,
-        )
+        user_rules = self._create_rule_components(model, self.schema, self.metadata)
 
-        self._end = datetime.now(timezone.utc)
         self.issue_list.trigger_warnings()
         if self.issue_list.has_errors:
             raise MultiValueError(self.issue_list.errors)
@@ -234,7 +257,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         schema: DMSSchema,
         metadata: DMSInputMetadata | None = None,
     ) -> DMSInputRules:
-        enum_by_container_property = self._create_enum_collections(schema.containers.values())
+        enum_by_container_property = self._create_enum_collections(self._all_containers_by_id.values())
         enum_collection_by_container_property = {
             key: enum_list[0].collection for key, enum_list in enum_by_container_property.items() if enum_list
         }
@@ -267,21 +290,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
             enum=[enum for enum_list in enum_by_container_property.values() for enum in enum_list] or None,
         )
 
-    @classmethod
-    def _create_default_metadata(
-        cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False
-    ) -> DMSInputMetadata:
-        now = datetime.now().replace(microsecond=0)
-        space = Counter(view.space for view in views).most_common(1)[0][0]
-        return DMSInputMetadata(
-            space=space,
-            external_id="Unknown",
-            version="0.1.0",
-            creator="Unknown",
-            created=now,
-            updated=now,
-        )
-
     def _create_dms_property(
         self,
         prop_id: str,
@@ -384,8 +392,11 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         elif isinstance(prop, dm.MappedPropertyApply):
             container_prop = self._container_prop_unsafe(cast(dm.MappedPropertyApply, prop))
             if isinstance(container_prop.type, dm.DirectRelation):
-                if prop.source is None or prop.source not in self._all_views_by_id:
+                if prop.source is None:
                     return DMSUnknownEntity()
+                elif prop.source not in self._all_views_by_id:
+                    self.issue_list.append(ResourceUnknownWarning(prop.source, "view", view_entity.as_id(), "view"))
+                    return ViewEntity.from_id(prop.source)
                 else:
                     return ViewEntity.from_id(prop.source)
             elif isinstance(container_prop.type, PropertyTypeWithUnit) and container_prop.type.unit:

cognite/neat/_rules/models/_base_rules.py

@@ -398,7 +398,10 @@ class SheetList(list, MutableSequence[T_SheetRow]):
 
     def _repr_html_(self) -> str:
         """Returns HTML representation of ResourceDict."""
-        return self.to_pandas(drop_na_columns=True)._repr_html_()  # type: ignore[operator]
+        df = self.to_pandas(drop_na_columns=True)
+        if "neatId" in df.columns:
+            df = df.drop(columns=["neatId"])
+        return df._repr_html_()  # type: ignore[operator]
 
     # Implemented to get correct type hints
     def __iter__(self) -> Iterator[T_SheetRow]:

cognite/neat/_rules/models/entities/_wrapped.py

@@ -9,7 +9,7 @@ from cognite.client import data_modeling as dm
 from cognite.client.data_classes.data_modeling import ContainerId, NodeId
 from pydantic import BaseModel, model_serializer, model_validator
 
-from ._single_value import ContainerEntity, DMSNodeEntity, Entity
+from ._single_value import ContainerEntity, DMSNodeEntity, Entity, ViewEntity
 
 
 @total_ordering
@@ -121,6 +121,8 @@ class DMSFilter(WrappedEntity):
             return HasDataFilter(
                 inner=[
                     ContainerEntity(space=entry["space"], externalId=entry["externalId"])
+                    if entry["type"] == "container"
+                    else ViewEntity(space=entry["space"], externalId=entry["externalId"], version=entry["version"])
                     for entry in body
                     if isinstance(entry, dict) and "space" in entry and "externalId" in entry
                 ]
@@ -161,21 +163,24 @@ class NodeTypeFilter(DMSFilter):
 
 class HasDataFilter(DMSFilter):
     name: ClassVar[str] = "hasData"
-    _inner_cls: ClassVar[type[ContainerEntity]] = ContainerEntity
-    inner: list[ContainerEntity] | None = None  # type: ignore[assignment]
+    _inner_cls: ClassVar[type[ContainerEntity | ViewEntity]] = ContainerEntity
+    inner: list[ContainerEntity | ViewEntity] | None = None  # type: ignore[assignment]
 
     def as_dms_filter(self, default: Collection[ContainerId] | None = None) -> dm.Filter:
         containers: list[ContainerId]
         if self.inner:
-            containers = [container.as_id() for container in self.inner]
+            containers = [item.as_id() for item in self.inner if isinstance(item, ContainerEntity)]
+            views = [item.as_id() for item in self.inner if isinstance(item, ViewEntity)]
         elif default:
             containers = list(default)
+            views = []
         else:
             raise ValueError("Empty hasData filter, please provide a default containers.")
 
         return dm.filters.HasData(
             # Sorting to ensure deterministic order
-            containers=sorted(containers, key=lambda container: container.as_tuple())  # type: ignore[union-attr]
+            containers=sorted(containers, key=lambda container: container.as_tuple()),  # type: ignore[union-attr]
+            views=sorted(views, key=lambda view: view.as_tuple()),  # type: ignore[union-attr]
         )