cognite-neat 1.0.17__tar.gz → 1.0.19__tar.gz

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/PKG-INFO

@@ -1,10 +1,22 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 1.0.17
+Version: 1.0.19
 Summary: Knowledge graph transformation
 Author: Nikola Vasiljevic, Anders Albert
 Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>
 License-Expression: Apache-2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Customer Service
+Classifier: Topic :: Database
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
 Requires-Dist: pandas>=1.5.3,<3.0.0
 Requires-Dist: cognite-sdk>=7.83.0,<8.0.0
 Requires-Dist: rdflib>=7.0.0,<8.0.0

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_config.py

@@ -101,6 +101,10 @@ class AlphaFlagConfig(ConfigModel):
         default=False,
         description="If enabled, Neat will attempt to automatically fix certain validation issues in the data model.",
     )
+    enable_experimental_validators: bool = Field(
+        default=False,
+        description="If enabled, Neat will run experimental validators that are still in alpha stage.",
+    )
 
     def __setattr__(self, key: str, value: Any) -> None:
         """Set attribute value or raise AttributeError."""

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_data_model/_analysis.py

@@ -1,9 +1,12 @@
+from collections import defaultdict
 from itertools import chain
-from typing import Literal, TypeAlias
+from typing import Literal, TypeAlias, TypeVar
 
+import networkx as nx
 from pyparsing import cached_property
 
 from cognite.neat._data_model._snapshot import SchemaSnapshot
+from cognite.neat._data_model.models.dms._constraints import RequiresConstraintDefinition
 from cognite.neat._data_model.models.dms._container import ContainerRequest
 from cognite.neat._data_model.models.dms._data_types import DirectNodeRelation
 from cognite.neat._data_model.models.dms._limits import SchemaLimits
@@ -26,6 +29,7 @@ from cognite.neat._utils.useful_types import ModusOperandi
 ViewsByReference: TypeAlias = dict[ViewReference, ViewRequest]
 ContainersByReference: TypeAlias = dict[ContainerReference, ContainerRequest]
 AncestorsByReference: TypeAlias = dict[ViewReference, set[ViewReference]]
+
 ReverseToDirectMapping: TypeAlias = dict[
     tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]
 ]
@@ -34,6 +38,8 @@ ConnectionEndNodeTypes: TypeAlias = dict[tuple[ViewReference, str], ViewReferenc
 
 ResourceSource = Literal["auto", "merged", "cdf", "both"]
 
+_NodeT = TypeVar("_NodeT", ContainerReference, ViewReference)
+
 
 class ValidationResources:
     def __init__(
@@ -378,3 +384,155 @@ class ValidationResources:
                         connection_end_node_types[(view_ref, prop_ref)] = property_.source
 
         return connection_end_node_types
+
+    @cached_property
+    def views_by_container(self) -> dict[ContainerReference, set[ViewReference]]:
+        """Get a mapping from containers to the views that use them.
+
+        Includes views from both the merged schema and all CDF views to capture
+        container-view relationships across the entire CDF environment.
+        Uses expanded views to include inherited properties.
+        """
+
+        # Include all unique views from merged and CDF
+        views_by_container: dict[ContainerReference, set[ViewReference]] = defaultdict(set)
+
+        # Include all unique views from merged and CDF
+        all_view_refs = set(self.merged.views.keys()) | set(self.cdf.views.keys())
+
+        for view_ref in all_view_refs:
+            # Use expanded view to include inherited properties
+            view = self.expand_view_properties(view_ref)
+            if not view:
+                continue
+
+            for container in view.used_containers:
+                views_by_container[container].add(view_ref)
+
+        return dict(views_by_container)
+
+    @cached_property
+    def containers_by_view(self) -> dict[ViewReference, set[ContainerReference]]:
+        """Get a mapping from views to the containers they use.
+
+        Includes views from both the merged schema and all CDF views.
+        Uses expanded views to include inherited properties.
+        """
+        containers_by_view: dict[ViewReference, set[ContainerReference]] = {}
+
+        # Include all unique views from merged and CDF
+        all_view_refs = set(self.merged.views.keys()) | set(self.cdf.views.keys())
+
+        for view_ref in all_view_refs:
+            # Use expanded view to include inherited properties
+            view = self.expand_view_properties(view_ref)
+            if view is not None:
+                containers_by_view[view_ref] = view.used_containers
+
+        return containers_by_view
+
+    def find_views_mapping_to_containers(self, containers: list[ContainerReference]) -> set[ViewReference]:
+        """Find views that map to all specified containers.
+
+        That is, the intersection of views that use each of the specified containers.
+
+        Args:
+            containers: List of containers to check
+
+        Returns:
+            Set of views that contain all the specified containers
+
+        Example:
+            Given views V1, V2, V3 and containers C1, C2:
+            - V1 uses containers {C1, C2}
+            - V2 uses containers {C1}
+            - V3 uses containers {C2}
+
+            find_views_mapping_to_containers([C1, C2]) returns {V1}
+            find_views_mapping_to_containers([C1]) returns {V1, V2}
+            find_views_mapping_to_containers([C2]) returns {V1, V3}
+        """
+        if not containers:
+            return set()
+
+        view_sets = [self.views_by_container.get(c, set()) for c in containers]
+        return set.intersection(*view_sets)
+
+    # =========================================================================
+    # Methods used for requires constraint validation
+    # =========================================================================
+
+    @cached_property
+    def requires_constraint_graph(self) -> nx.DiGraph:
+        """Build a directed graph of container requires constraints.
+
+        Nodes are ContainerReferences, edges represent requires constraints.
+        An edge A → B means container A requires container B.
+
+        Includes containers from both merged schema and CDF
+        """
+        graph: nx.DiGraph = nx.DiGraph()
+
+        for container_ref in self.cdf.containers:
+            graph.add_node(container_ref)
+        for container_ref in self.merged.containers:
+            graph.add_node(container_ref)
+
+        # Add edges for requires constraints from all known containers
+        for container_ref in graph.nodes():
+            container = self.select_container(container_ref)
+            if not container or not container.constraints:
+                continue
+            for constraint in container.constraints.values():
+                if not isinstance(constraint, RequiresConstraintDefinition):
+                    continue
+                graph.add_edge(container_ref, constraint.require)
+
+        return graph
+
+    @staticmethod
+    def forms_directed_path(nodes: set[_NodeT], graph: nx.DiGraph) -> bool:
+        """Check if nodes form an uninterrupted directed path in the graph.
+
+        Returns True if there exists a node that can reach all other nodes via
+        directed edges in the graph.
+
+        Args:
+            nodes: Set of nodes to check
+            graph: Directed graph containing the nodes
+
+        Returns:
+            True if nodes form a directed path (one node reaches all others)
+
+        Example:
+            Given nodes N1, N2, N3 with edges:
+            - N1 -> N2
+            - N2 -> N3
+
+            forms_directed_path({N1, N2, N3}) returns True (N1 reaches all others)
+            forms_directed_path({N2, N3}) returns True (N2 reaches N3)
+            forms_directed_path({N1, N3}) returns False (N1 can't reach N3 without N2)
+        """
+        if len(nodes) <= 1:
+            return True
+
+        for candidate in nodes:
+            others = nodes - {candidate}
+            if others.issubset(nx.descendants(graph, candidate)):
+                return True
+
+        return False
+
+    @cached_property
+    def requires_constraint_cycles(self) -> list[set[ContainerReference]]:
+        """Find all cycles in the requires constraint graph using Tarjan's algorithm.
+
+        Uses strongly connected components (SCC) to identify cycles efficiently.
+        An SCC with more than one node indicates a cycle.
+
+        Returns:
+            List of sets, where each set contains the containers involved in a cycle.
+        """
+        sccs = nx.strongly_connected_components(self.requires_constraint_graph)
+        # Only SCCs with more than one node represent cycles
+        return [scc for scc in sccs if len(scc) > 1]

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_data_model/_constants.py

@@ -59,7 +59,14 @@ COGNITE_CONCEPTS: tuple[str, ...] = (
     *COGNITE_CONCEPTS_3D,
 )
 
-COGNITE_SPACES = (CDF_CDM_SPACE,)
+COGNITE_SPACES = (
+    CDF_CDM_SPACE,
+    "cdf_360_image_schema",
+    "cdf_3d_schema",
+    "cdf_apm",
+    "cdf_apps_shared",
+    "cdf_cdm_3d",
+)
 
 # Defaults from https://docs.cognite.com/cdf/dm/dm_reference/dm_limits_and_restrictions#list-size-limits
 

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_data_model/_snapshot.py

@@ -76,6 +76,7 @@ class SchemaSnapshot(BaseModel, extra="ignore"):
         for container_ref, container in merged.containers.items():
             if cdf_container := cdf.containers.get(container_ref):
                 container.properties = {**cdf_container.properties, **container.properties}
+                container.constraints = {**(cdf_container.constraints or {}), **(container.constraints or {})} or None
 
         return merged
 

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_data_model/deployer/deployer.py

@@ -4,16 +4,15 @@ from functools import partial
 from typing import cast
 
 from cognite.neat._client import NeatClient
+from cognite.neat._data_model._constants import COGNITE_SPACES
 from cognite.neat._data_model._shared import OnSuccessResultProducer
 from cognite.neat._data_model._snapshot import SchemaSnapshot
 from cognite.neat._data_model.models.dms import (
-    ContainerReference,
     ContainerRequest,
     DataModelBody,
     RequestSchema,
     T_DataModelResource,
     T_ResourceId,
-    ViewReference,
 )
 from cognite.neat._utils.collection import chunker_sequence
 from cognite.neat._utils.http_client import (
@@ -130,7 +129,13 @@ class SchemaDeployer(OnSuccessResultProducer):
     def create_deployment_plan(self, snapshot: SchemaSnapshot, data_model: RequestSchema) -> ResourceDeploymentPlanList:
         return ResourceDeploymentPlanList(
             [
-                self._create_resource_plan(snapshot.spaces, data_model.spaces, "spaces", SpaceDiffer()),
+                self._create_resource_plan(
+                    snapshot.spaces,
+                    data_model.spaces,
+                    "spaces",
+                    SpaceDiffer(),
+                    skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
+                ),
                 self._create_resource_plan(
                     snapshot.containers,
                     data_model.containers,
@@ -150,7 +155,11 @@ class SchemaDeployer(OnSuccessResultProducer):
                     skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
                 ),
                 self._create_resource_plan(
-                    snapshot.data_model, [data_model.data_model], "datamodels", DataModelDiffer()
+                    snapshot.data_model,
+                    [data_model.data_model],
+                    "datamodels",
+                    DataModelDiffer(),
+                    skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
                 ),
             ]
         )
@@ -234,7 +243,7 @@ class SchemaDeployer(OnSuccessResultProducer):
         return modified_diffs
 
     @classmethod
-    def _skip_resource(cls, resource_id: ContainerReference | ViewReference, model_space: str) -> str | None:
+    def _skip_resource(cls, resource_id: T_ResourceId, model_space: str) -> str | None:
         """Checks if a resource should be skipped based on its space.
 
         Args:
@@ -244,8 +253,13 @@ class SchemaDeployer(OnSuccessResultProducer):
         Returns:
             A reason for skipping if the resource space does not match the model space, otherwise None.
         """
-        if resource_id.space != model_space:
-            return f"Skipping resource in space '{resource_id.space}' not matching data model space '{model_space}'."
+        if resource_id.space in COGNITE_SPACES:
+            return f"Skipping resource as it is in the reserved Cognite space '{resource_id.space}'."
+        elif resource_id.space != model_space:
+            return (
+                f"Skipping resource as it is in the space '{resource_id.space}'"
+                f" not matching data model space '{model_space}'."
+            )
         return None
 
     def should_proceed_to_deploy(self, plan: Sequence[ResourceDeploymentPlan]) -> bool:

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_data_model/models/dms/_view_filter.py

@@ -203,7 +203,10 @@ FilterTypes: TypeAlias = Literal[
     "not",
 ]
 
+LegacyFilterTypes: TypeAlias = Literal["invalid"]
+
 AVAILABLE_FILTERS: frozenset[str] = frozenset(get_args(FilterTypes))
+LEGACY_FILTERS: frozenset[str] = frozenset(get_args(LegacyFilterTypes))
 
 
 def _move_filter_key(value: Any) -> Any:
@@ -246,13 +249,13 @@ def _move_filter_key(value: Any) -> Any:
         }
     }
     """
+    # legacy filter which we want to ignore
+    if _is_legacy_filter(value):
+        return None
     if not isinstance(value, dict):
         return value
     if len(value) != 1:
         raise ValueError("Filter data must have exactly one key.")
-    # legacy filter which we want to ignore
-    if "invalid" in value:
-        return None
     if "filterType" in value:
         # Already in the correct format
         return value
@@ -283,6 +286,25 @@ def _move_filter_key(value: Any) -> Any:
         return value
 
 
+def _is_legacy_filter(filter_obj: Any) -> bool:
+    """Check if filter is a legacy filter no longer supported by DMS API"""
+
+    def traverse(obj: Any) -> bool:
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                if key in LEGACY_FILTERS:
+                    return True
+                if traverse(value):
+                    return True
+        elif isinstance(obj, list):
+            for item in obj:
+                if traverse(item):
+                    return True
+        return False
+
+    return traverse(filter_obj)
+
+
 Filter = Annotated[dict[FilterTypes, FilterData] | None, BeforeValidator(_move_filter_key)]
 
 FilterAdapter: TypeAdapter[Filter] = TypeAdapter(Filter)

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_data_model/validation/dms/_base.py

@@ -10,6 +10,7 @@ class DataModelValidator(ABC):
 
     code: ClassVar[str]
     issue_type: ClassVar[type[ConsistencyError] | type[Recommendation]]
+    alpha: ClassVar[bool] = False
 
     def __init__(
         self,

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_data_model/validation/dms/_orchestrator.py

@@ -21,6 +21,7 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
         limits: SchemaLimits,
         modus_operandi: ModusOperandi = "additive",
         can_run_validator: Callable[[str, type], bool] | None = None,
+        enable_alpha_validators: bool = False,
     ) -> None:
         super().__init__()
         self._cdf_snapshot = cdf_snapshot
@@ -28,6 +29,7 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
         self._modus_operandi = modus_operandi
         self._can_run_validator = can_run_validator or (lambda code, issue_type: True)  # type: ignore
         self._has_run = False
+        self._enable_alpha_validators = enable_alpha_validators
 
     def run(self, request_schema: RequestSchema) -> None:
         """Run quality assessment on the DMS data model."""
@@ -41,6 +43,8 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
 
         # Run validators
         for validator in validators:
+            if validator.alpha and not self._enable_alpha_validators:
+                continue
             if self._can_run_validator(validator.code, validator.issue_type):
                 self._issues.extend(validator.run())
 

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_session/_physical.py

@@ -106,8 +106,9 @@ class ReadPhysicalDataModel:
         on_success = DmsDataModelValidation(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
-            limits=self._store.limits,
+            limits=self._store.cdf_limits,
             can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
         )
         return self._store.read_physical(reader, on_success)
 
@@ -134,8 +135,9 @@ class ReadPhysicalDataModel:
         on_success = DmsDataModelValidation(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
-            limits=self._store.limits,
+            limits=self._store.cdf_limits,
             can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
         )
         return self._store.read_physical(reader, on_success)
 
@@ -153,8 +155,9 @@ class ReadPhysicalDataModel:
         on_success = DmsDataModelValidation(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
-            limits=self._store.limits,
+            limits=self._store.cdf_limits,
             can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
         )
 
         return self._store.read_physical(reader, on_success)
@@ -175,8 +178,9 @@ class ReadPhysicalDataModel:
         on_success = DmsDataModelValidation(
             modus_operandi=self._config.modeling.mode,
             cdf_snapshot=self._store.cdf_snapshot,
-            limits=self._store.limits,
+            limits=self._store.cdf_limits,
             can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
         )
 
         return self._store.read_physical(reader, on_success)

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_session/_result/_deployment/_physical/_changes.py

@@ -101,6 +101,7 @@ class SerializedResourceChange(BaseModel):
             severity=resource.severity.name,
             resource_id=str(resource.resource_id),
             changes=changes,
+            message=resource.message,
         )
 
     @classmethod

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/cognite/neat/_store/_store.py

@@ -34,9 +34,21 @@ class NeatStore:
         self._client = client
         self._config = config
 
-        # Caching CDF Schema limits and snapshot
-        self.limits = SchemaLimits.from_api_response(self._client.statistics.project())
-        self.cdf_snapshot = SchemaSnapshot.fetch_entire_cdf(self._client)
+        # Placeholder for CDF schema and limit snapshot
+        self._cdf_snapshot: SchemaSnapshot | None = None
+        self._cdf_limits: SchemaLimits | None = None
+
+    @property
+    def cdf_limits(self) -> SchemaLimits:
+        if not self._cdf_limits:
+            self._cdf_limits = SchemaLimits.from_api_response(self._client.statistics.project())
+        return self._cdf_limits
+
+    @property
+    def cdf_snapshot(self) -> SchemaSnapshot:
+        if not self._cdf_snapshot:
+            self._cdf_snapshot = SchemaSnapshot.fetch_entire_cdf(self._client)
+        return self._cdf_snapshot
 
     def read_physical(self, reader: DMSImporter, on_success: OnSuccess | None = None) -> None:
         """Read object from the store"""
@@ -81,7 +93,7 @@ class NeatStore:
             and not on_success.options.dry_run
         ):
             # Update CDF snapshot after successful deployment
-            self.cdf_snapshot = SchemaSnapshot.fetch_entire_cdf(self._client)
+            self._cdf_snapshot = SchemaSnapshot.fetch_entire_cdf(self._client)
 
     def _gather_data_model(self, writer: DMSExporter) -> PhysicalDataModel:
         """Gather the current physical data model from the store

cognite_neat-1.0.19/cognite/neat/_version.py

@@ -0,0 +1,2 @@
+__version__ = "1.0.19"
+__engine__ = "^2.0.4"

{cognite_neat-1.0.17 → cognite_neat-1.0.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite-neat"
-version = "1.0.17"
+version = "1.0.19"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [
@@ -9,6 +9,20 @@ authors = [
 ]
 license = "Apache-2.0"
 requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: Customer Service",
+    "Topic :: Database",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Operating System :: OS Independent",
+]
 dependencies = [
     "pandas>=1.5.3, <3.0.0",
     "cognite-sdk>=7.83.0, <8.0.0",
@@ -123,8 +137,8 @@ target-version = "py310"
 exclude = ["scripts"]
 fix = true
 
-[tool.coverage.report]
-show_missing = true
+[tool.coverage.run]
+source = ["cognite.neat"]
 omit = [
     # This contains the legacy code. We want to exclude it from coverage reports
     # such that we can increase the coverage threshold for the new code.
@@ -134,6 +148,20 @@ omit = [
     "cognite/neat/_session/_usage_analytics/*",
 ]
 
+[tool.coverage.report]
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "raise RuntimeError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@abstractmethod",
+    "from typing_extensions import Self",
+]
+
 [tool.mypy]
 pretty = true
 check_untyped_defs = true

cognite_neat-1.0.17/cognite/neat/_version.py

@@ -1,2 +0,0 @@
-__version__ = "1.0.17"
-__engine__ = "^2.0.4"