cognite-toolkit 0.6.100__tar.gz → 0.6.101__tar.gz

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.100
+Version: 0.6.101
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/cdf.toml

@@ -32,7 +32,7 @@ dev = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.100"
+version = "0.6.101"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -1,13 +1,18 @@
+from collections import Counter
 from collections.abc import Sequence
 from functools import partial
 from pathlib import Path
 
 from cognite.client.data_classes._base import T_CogniteResource
+from cognite.client.data_classes.data_modeling import (
+    ViewId,
+)
 from pydantic import ValidationError
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.cruds import ViewCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio import (
     T_Selector,
@@ -17,6 +22,7 @@ from cognite_toolkit._cdf_tk.storageio import (
 )
 from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
 from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
+from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
@@ -86,8 +92,49 @@ class UploadCommand(ToolkitCommand):
 
         self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
 
+        data_files_by_selector = self._topological_sort_if_instance_selector(data_files_by_selector, client)
+
         self._upload_data(data_files_by_selector, client, dry_run, input_dir, console, verbose)
 
+    def _topological_sort_if_instance_selector(
+        self, data_files_by_selector: dict[Selector, list[Path]], client: ToolkitClient
+    ) -> dict[Selector, list[Path]]:
+        """Topologically sorts InstanceSpaceSelectors (if they are present) to determine the order of upload based on container dependencies from the views.
+
+        Args:
+            data_files_by_selector: A dictionary mapping selectors to their data files.
+            client: The cognite client to use for the upload.
+
+        Returns:
+            A dictionary mapping selectors to their data files with necessary preprocessing.
+        """
+        counts = Counter(type(selector) for selector in data_files_by_selector.keys())
+        if counts[InstanceSpaceSelector] <= 1:
+            return data_files_by_selector
+
+        selector_by_view_id: dict[ViewId, Selector] = {}
+        for selector in data_files_by_selector:
+            if isinstance(selector, InstanceSpaceSelector) and selector.view is not None:
+                selector_by_view_id[selector.view.as_id()] = selector
+
+        view_dependencies = ViewCRUD.create_loader(client).topological_sort_container_constraints(
+            list(selector_by_view_id.keys())
+        )
+        prepared_selectors: dict[Selector, list[Path]] = {}
+
+        # Reorder selectors according to the dependency-sorted view list
+        for view_id in view_dependencies:
+            selector = selector_by_view_id[view_id]
+            prepared_selectors[selector] = data_files_by_selector[selector]
+
+        # Preserve selectors that aren't affected by view dependencies
+        # (e.g., raw tables, time series, non-view instance data)
+        for selector in data_files_by_selector.keys():
+            if selector not in prepared_selectors:
+                prepared_selectors[selector] = data_files_by_selector[selector]
+
+        return prepared_selectors
+
     def _find_data_files(
         self,
         input_dir: Path,

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py

@@ -40,6 +40,7 @@ from cognite.client.data_classes.data_modeling import (
     DataModelApply,
     DataModelApplyList,
     DataModelList,
+    DirectRelation,
     Edge,
     EdgeApply,
     EdgeApplyList,
@@ -51,6 +52,7 @@ from cognite.client.data_classes.data_modeling import (
     NodeApplyList,
     NodeApplyResultList,
     NodeList,
+    RequiresConstraint,
     Space,
     SpaceApply,
     SpaceApplyList,
@@ -261,6 +263,16 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
     yaml_cls = ContainerYAML
     _doc_url = "Containers/operation/ApplyContainers"
 
+    def __init__(
+        self,
+        client: ToolkitClient,
+        build_dir: Path | None,
+        console: Console | None = None,
+        topological_sort_implements: bool = False,
+    ) -> None:
+        super().__init__(client, build_dir, console)
+        self._container_by_id: dict[ContainerId, Container] = {}
+
     @property
     def display_name(self) -> str:
         return "containers"
@@ -427,6 +439,71 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
             ):
                 yield instances.as_ids()
 
+    def _lookup_containers(self, container_ids: Sequence[ContainerId]) -> dict[ContainerId, Container]:
+        ids_to_lookup = [container_id for container_id in container_ids if container_id not in self._container_by_id]
+        if ids_to_lookup:
+            retrieved_containers = self.client.data_modeling.containers.retrieve(ids_to_lookup)
+            for container in retrieved_containers:
+                self._container_by_id[container.as_id()] = container
+        if missing_container_ids := set(container_ids) - set(self._container_by_id.keys()):
+            MediumSeverityWarning(
+                f"Containers {missing_container_ids} not found or you don't have permission to access them."
+            ).print_warning(console=self.console)
+        return {
+            container_id: self._container_by_id[container_id]
+            for container_id in container_ids
+            if container_id in self._container_by_id
+        }
+
+    def _find_direct_container_dependencies(
+        self, container_ids: Sequence[ContainerId]
+    ) -> dict[ContainerId, set[ContainerId]]:
+        containers_by_id = self._lookup_containers(container_ids)
+        container_dependencies: dict[ContainerId, set[ContainerId]] = defaultdict(set)
+        for container_id, container in containers_by_id.items():
+            for constraint in container.constraints.values():
+                if not isinstance(constraint, RequiresConstraint):
+                    continue
+                container_dependencies[container_id].add(constraint.require)
+            for property in container.properties.values():
+                if not isinstance(property.type, DirectRelation) or property.type.container is None:
+                    continue
+                container_dependencies[container_id].add(property.type.container)
+        return container_dependencies
+
+    def _propagate_indirect_container_dependencies(
+        self, container_dependencies_by_id: dict[ContainerId, set[ContainerId]], dependants: Sequence[ContainerId]
+    ) -> dict[ContainerId, set[ContainerId]]:
+        """Propagate indirect container dependencies using a recursive approach.
+
+        Args:
+            container_dependencies_by_id: Mapping of container IDs to their direct dependencies
+            dependants: Chain of dependant containers to propagate dependencies to
+
+        Returns:
+            Updated dictionary mapping each container ID to all its direct and indirect dependencies
+        """
+        current_container_id = dependants[0]
+        dependencies_to_propagate: set[ContainerId] = set()
+        for container_dependency in container_dependencies_by_id[current_container_id]:
+            if container_dependency in container_dependencies_by_id:
+                # If already processed, propagate its dependencies to current container instead of revisiting it
+                dependencies_to_propagate.update(container_dependencies_by_id[container_dependency])
+                continue
+            self._propagate_indirect_container_dependencies(
+                container_dependencies_by_id, [container_dependency, *dependants]
+            )
+        container_dependencies_by_id[current_container_id].update(dependencies_to_propagate)
+        return container_dependencies_by_id
+
+    def _find_direct_and_indirect_container_dependencies(
+        self, container_ids: Sequence[ContainerId]
+    ) -> dict[ContainerId, set[ContainerId]]:
+        container_dependencies_by_id = self._find_direct_container_dependencies(container_ids)
+        for container_id in list(container_dependencies_by_id.keys()):
+            self._propagate_indirect_container_dependencies(container_dependencies_by_id, [container_id])
+        return container_dependencies_by_id
+
     @staticmethod
     def _chunker(seq: Sequence, size: int) -> Iterable[Sequence]:
         return (seq[pos : pos + size] for pos in range(0, len(seq), size))
@@ -558,7 +635,9 @@ class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View, ViewApplyList, ViewList]):
             # We sort the implements in topological order to ensure that the child view get the order grandparent,
             # parent, such that the parent's source is used.
             try:
-                dumped["implements"] = [view_id.dump() for view_id in self.topological_sort(resource.implements)]
+                dumped["implements"] = [
+                    view_id.dump() for view_id in self.topological_sort_implements(resource.implements)
+                ]
             except ToolkitCycleError as e:
                 warning = MediumSeverityWarning(f"Failed to sort implements for view {resource.as_id()}: {e}")
                 warning.print_warning(console=self.console)
@@ -691,14 +770,31 @@ class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View, ViewApplyList, ViewList]):
                 readonly_properties.add(property_identifier)
         return readonly_properties
 
-    def topological_sort(self, view_ids: list[ViewId]) -> list[ViewId]:
+    def _build_view_implements_dependencies(
+        self, view_by_ids: dict[ViewId, View], include: set[ViewId] | None = None
+    ) -> dict[ViewId, set[ViewId]]:
+        """Build a dependency graph based on view implements relationships.
+
+        Args:
+            view_by_ids: Mapping of view IDs to View objects
+            include: Optional set of view IDs to include in the dependencies, if None, all views are included.
+
+        Returns:
+            Dictionary mapping each view ID to the set of view IDs it depends on (implements)
+        """
+        dependencies: dict[ViewId, set[ViewId]] = {}
+        for view_id, view in view_by_ids.items():
+            dependencies[view_id] = set()
+            for implemented_view_id in view.implements or []:
+                if include is None or implemented_view_id in include:
+                    dependencies[view_id].add(implemented_view_id)
+        return dependencies
+
+    def topological_sort_implements(self, view_ids: list[ViewId]) -> list[ViewId]:
         """Sorts the views in topological order based on their implements and through properties."""
         view_by_ids = self._lookup_views(view_ids)
-        parents_by_child: dict[ViewId, set[ViewId]] = {}
-        for child, view in view_by_ids.items():
-            parents_by_child[child] = set()
-            for parent in view.implements or []:
-                parents_by_child[child].add(parent)
+        parents_by_child = self._build_view_implements_dependencies(view_by_ids)
+
         try:
             sorted_views = list(TopologicalSorter(parents_by_child).static_order())
         except CycleError as e:
@@ -708,6 +804,54 @@ class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View, ViewApplyList, ViewList]):
 
         return sorted_views
 
+    def topological_sort_container_constraints(self, view_ids: list[ViewId]) -> list[ViewId]:
+        """Sorts the views in topological order based on their container constraints."""
+
+        view_by_ids = self._lookup_views(view_ids)
+        if missing_view_ids := set(view_ids) - set(view_by_ids.keys()):
+            MediumSeverityWarning(
+                f"Views {missing_view_ids} not found or you don't have permission to access them, skipping dependency check."
+            ).print_warning(console=self.console)
+            return view_ids
+
+        view_to_containers: dict[ViewId, set[ContainerId]] = {}
+        container_to_views: defaultdict[ContainerId, set[ViewId]] = defaultdict(set)
+        for view_id, view in view_by_ids.items():
+            view_to_containers[view_id] = view.referenced_containers()
+            for container_id in view_to_containers[view_id]:
+                container_to_views[container_id].add(view_id)
+
+        container_crud = ContainerCRUD.create_loader(self.client)
+        container_dependencies_by_id = container_crud._find_direct_and_indirect_container_dependencies(
+            list(container_to_views.keys())
+        )
+
+        # First, add view dependencies based on implements relationships
+        view_dependencies = self._build_view_implements_dependencies(view_by_ids, set(view_to_containers.keys()))
+
+        # Then, add view dependencies based on mapped container constraints
+        for view_id, mapped_containers in view_to_containers.items():
+            for container_id in mapped_containers:
+                # Get all containers this container depends on
+                if container_id not in container_dependencies_by_id:
+                    continue
+                for required_container in container_dependencies_by_id[container_id]:
+                    if required_container not in container_to_views:
+                        continue
+                    # If this view already implements the required container, the requirement is self-satisfied
+                    # and we don't need to depend on other views that also implement it (they are peers).
+                    if required_container in mapped_containers:
+                        continue
+                    # This view doesn't implement the required container, so depend on all views that do
+                    view_dependencies[view_id].update(container_to_views[required_container])
+        try:
+            sorted_views = list(TopologicalSorter(view_dependencies).static_order())
+        except CycleError as e:
+            raise ToolkitCycleError(
+                f"Failed to sort views topologically. This likely due to a cycle in implements. {e.args[1]}"
+            )
+        return sorted_views
+
 
 @final
 class DataModelCRUD(ResourceCRUD[DataModelId, DataModelApply, DataModel, DataModelApplyList, DataModelList]):

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.100
+      image: cognite/toolkit:0.6.101
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.100
+      image: cognite/toolkit:0.6.101
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.100"
+version = "0.6.101"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.101/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.101"

cognite_toolkit-0.6.101/last_changelog_entry.md

@@ -0,0 +1,19 @@
+## cdf 
+
+### Added
+
+- Added automatic dependency resolution for data model instances based
+on container constraints when `data upload`
+- Added a new topological sorting method
+`topological_sort_container_constraints` to `ViewCRUD` based on
+container constraints, in addition to the existing method that considers
+`implements`
+
+### Changed
+
+- Renamed `topological_sort` in `ViewCRUD` to
+`topological_sort_implements` to distinguish between the new method.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.101/last_git_message.txt

@@ -0,0 +1,40 @@
+[CDF-25328] Consider container constraints when uploading instances (#2164)
+
+# Description
+
+This PR adds automatic dependency resolution for data model instances
+during upload when InstanceSpaceSelectors are used. The added
+functionality analyzes requires container constraints and direct
+relations with a required type constraint to determine the necessary
+upload order, ensuring referenced entities are created before entities
+that reference them.
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Added
+
+- Added automatic dependency resolution for data model instances based
+on container constraints when `data upload`
+- Added a new topological sorting method
+`topological_sort_container_constraints` to `ViewCRUD` based on
+container constraints, in addition to the existing method that considers
+`implements`
+
+### Changed
+
+- Renamed `topological_sort` in `ViewCRUD` to
+`topological_sort_implements` to distinguish between the new method.
+
+## templates
+
+No changes.
+
+---------
+
+Co-authored-by: Magnus Schjølberg <magnus.schjolberg@cognitedata.com>
+Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com>

cognite_toolkit-0.6.101/last_version.txt

@@ -0,0 +1 @@
+0.6.100

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.100"
+version = "0.6.101"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/tests/test_unit/approval_client/config.py

@@ -381,7 +381,7 @@ API_RESOURCES = [
             "delete": [Method(api_class_method="delete", mock_class_method="delete_data_modeling")],
             "retrieve": [
                 Method(api_class_method="list", mock_class_method="return_values"),
-                Method(api_class_method="retrieve", mock_class_method="return_values"),
+                Method(api_class_method="retrieve", mock_class_method="data_model_retrieve"),
             ],
         },
     ),

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/tests/test_unit/test_cdf_tk/test_commands/test_upload.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 import pytest
 import respx
+from cognite.client.data_classes import data_modeling as dm
 from cognite.client.data_classes.raw import RowWrite, Table, TableList
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
@@ -13,8 +14,15 @@ from cognite_toolkit._cdf_tk.commands import UploadCommand
 from cognite_toolkit._cdf_tk.constants import DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.cruds import RawTableCRUD
 from cognite_toolkit._cdf_tk.storageio import RawIO
-from cognite_toolkit._cdf_tk.storageio.selectors import RawTableSelector, SelectedTable
+from cognite_toolkit._cdf_tk.storageio.selectors import (
+    InstanceSpaceSelector,
+    RawTableSelector,
+    SelectedTable,
+    SelectedView,
+    Selector,
+)
 from cognite_toolkit._cdf_tk.utils.fileio import NDJsonWriter, Uncompressed
+from tests.test_unit.approval_client import ApprovalToolkitClient
 
 
 @pytest.fixture
@@ -142,3 +150,45 @@ class TestUploadCommand:
             client.raw.rows.insert.assert_not_called()
             client.raw.databases.create.assert_not_called()
             client.raw.tables.create.assert_not_called()
+
+    def test_instance_selector_topological_sorts_and_preserves_selectors(
+        self,
+        toolkit_client_approval: ApprovalToolkitClient,
+        cognite_core_no_3D: dm.DataModel,
+        cognite_core_containers_no_3D: dm.ContainerList,
+    ) -> None:
+        """Test that _topological_sort_if_instance_selector sorts instance selectors by dependencies and preserves non-instance selectors."""
+        cmd = UploadCommand(silent=True, skip_tracking=True)
+        toolkit_client_approval.append(dm.View, cognite_core_no_3D.views)
+        toolkit_client_approval.append(dm.Container, cognite_core_containers_no_3D)
+
+        data_files_by_selector: dict[Selector, list[Path]] = {}
+        selector_by_view_external_id: dict[str, Selector] = {}
+
+        # Create instance selectors in reverse dependency order
+        for view_external_id in ["CogniteAsset", "CogniteSourceSystem"]:
+            selector = InstanceSpaceSelector(
+                instance_space="cdf_cdm",
+                view=SelectedView(space="cdf_cdm", external_id=view_external_id, version="v1"),
+                type="instanceSpace",
+            )
+            data_files_by_selector[selector] = [Path(f"dummy_{view_external_id}.json")]  # type: ignore[reportUnhashable]
+            selector_by_view_external_id[view_external_id] = selector
+
+        # Add a non-instance selector
+        raw_selector = RawTableSelector(
+            table=SelectedTable(db_name="test_db", table_name="test_table"), type="rawTable"
+        )
+        data_files_by_selector[raw_selector] = [Path("raw_data.json")]  # type: ignore[reportUnhashable]
+
+        result = cmd._topological_sort_if_instance_selector(data_files_by_selector, toolkit_client_approval.mock_client)
+
+        # Verify instance selectors are sorted by dependencies
+        result_keys = list(result.keys())
+        assert result_keys.index(selector_by_view_external_id["CogniteSourceSystem"]) < result_keys.index(
+            selector_by_view_external_id["CogniteAsset"]
+        ), f"CogniteSourceSystem should come before CogniteAsset due to dependencies, but got order {result_keys}"
+
+        # Verify non-instance selectors are preserved
+        assert raw_selector in result, f"Raw selector should be preserved, not found in {result_keys}"
+        assert len(result) == 3, f"Expected 3 selectors, got {len(result)}: {result_keys}"

{cognite_toolkit-0.6.100 → cognite_toolkit-0.6.101}/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py

@@ -198,7 +198,7 @@ dml: model.graphql
 
 
 @pytest.fixture()
-def parent_grandparent_view() -> dm.ViewlList:
+def parent_grandparent_view() -> dm.ViewList:
     return dm.ViewList(
         [
             dm.View(
@@ -240,7 +240,7 @@ class TestViewLoader:
         with monkeypatch_toolkit_client() as client:
             client.data_modeling.views.retrieve.return_value = parent_grandparent_view
             loader = ViewCRUD(client, Path("build_dir"), None, topological_sort_implements=True)
-            actual = loader.topological_sort(
+            actual = loader.topological_sort_implements(
                 [dm.ViewId("space", "Parent", "v1"), dm.ViewId("space", "GrandParent", "v1")]
             )
 
@@ -252,6 +252,8 @@ class TestViewLoader:
         with monkeypatch_toolkit_client() as client, pytest.raises(ToolkitCycleError) as exc_info:
             client.data_modeling.views.retrieve.return_value = parent_grandparent_view
             loader = ViewCRUD(client, Path("build_dir"), None, topological_sort_implements=True)
-            loader.topological_sort([dm.ViewId("space", "Parent", "v1"), dm.ViewId("space", "GrandParent", "v1")])
+            loader.topological_sort_implements(
+                [dm.ViewId("space", "Parent", "v1"), dm.ViewId("space", "GrandParent", "v1")]
+            )
 
         assert "cycle in implements" in str(exc_info.value)