cognite-toolkit 0.6.83__py3-none-any.whl → 0.6.84__py3-none-any.whl

cognite_toolkit/_cdf_tk/client/_toolkit_client.py

@@ -1,6 +1,7 @@
 from typing import cast
 
 from cognite.client import CogniteClient
+from rich.console import Console
 
 from .api.canvas import CanvasAPI
 from .api.charts import ChartsAPI
@@ -24,11 +25,12 @@ class ToolkitClient(CogniteClient):
     def __init__(self, config: ToolkitClientConfig | None = None, enable_set_pending_ids: bool = False) -> None:
         super().__init__(config=config)
         toolkit_config = ToolkitClientConfig.from_client_config(self.config)
+        self.console = Console()
         self.search = SearchAPI(self._config, self._API_VERSION, self)
         self.robotics = RoboticsAPI(self._config, self._API_VERSION, self)
         self.dml = DMLAPI(self._config, self._API_VERSION, self)
         self.verify = VerifyAPI(self._config, self._API_VERSION, self)
-        self.lookup = LookUpGroup(self._config, self._API_VERSION, self)
+        self.lookup = LookUpGroup(self._config, self._API_VERSION, self, self.console)
         self.functions: ExtendedFunctionsAPI = ExtendedFunctionsAPI(toolkit_config, self._API_VERSION, self)
         self.data_modeling: ExtendedDataModelingAPI = ExtendedDataModelingAPI(self._config, self._API_VERSION, self)
         if enable_set_pending_ids:

cognite_toolkit/_cdf_tk/client/api/lookup.py

@@ -17,11 +17,12 @@ from cognite.client.data_classes.capabilities import (
 )
 from cognite.client.exceptions import CogniteAPIError
 from cognite.client.utils.useful_types import SequenceNotStr
+from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client.api_client import ToolkitAPI
 from cognite_toolkit._cdf_tk.constants import DRY_RUN_ID
-from cognite_toolkit._cdf_tk.exceptions import ResourceRetrievalError
 from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
+from cognite_toolkit._cdf_tk.utils import humanize_collection
 
 if TYPE_CHECKING:
     from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
@@ -30,17 +31,20 @@ if TYPE_CHECKING:
 class LookUpAPI(ToolkitAPI, ABC):
     dry_run_id: int = DRY_RUN_ID
 
-    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: "ToolkitClient") -> None:
+    def __init__(
+        self, config: ClientConfig, api_version: str | None, cognite_client: "ToolkitClient", console: Console
+    ) -> None:
         super().__init__(config, api_version, cognite_client)
-        self._cache: dict[str, int] = {}
-        self._reverse_cache: dict[int, str] = {}
+        self._console = console
+        self._cache: dict[str, int | None] = {}
+        self._reverse_cache: dict[int, str | None] = {}
 
     @property
     def resource_name(self) -> str:
         return type(self).__name__.removesuffix("LookUpAPI")
 
     @overload
-    def id(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int: ...
+    def id(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int | None: ...
 
     @overload
     def id(
@@ -49,39 +53,61 @@ class LookUpAPI(ToolkitAPI, ABC):
 
     def id(
         self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False, allow_empty: bool = False
-    ) -> int | list[int]:
+    ) -> int | None | list[int]:
+        """Lookup internal IDs for given external IDs.
+
+        Args:
+            external_id: A string external ID or a sequence of string external IDs to look up.
+            is_dry_run: If True, return a predefined dry run ID for missing external IDs instead of raising an error.
+            allow_empty: If True, allow empty string external IDs and return 0 for them.
+
+        Returns:
+            The corresponding internal ID as an integer if a single external ID is provided,
+            or a list of internal IDs if a sequence of external IDs is provided.
+        """
         ids = [external_id] if isinstance(external_id, str) else external_id
-        missing = [id for id in ids if id not in self._cache]
-        if allow_empty and "" in missing:
+        need_lookup = [id for id in ids if id not in self._cache]
+        if allow_empty and "" in need_lookup:
             # Note we do not want to put empty string in the cache. It is a special case that
             # as of 01/02/2025 only applies to LocationFilters
-            missing.remove("")
-        if missing:
-            try:
-                lookup = self._id(missing)
-            except CogniteAPIError as e:
-                if 400 <= e.code < 500:
-                    missing_capabilities = self._toolkit_client.verify.authorization(self._read_acl())
-                    if missing_capabilities:
-                        raise self._toolkit_client.verify.create_error(
-                            missing_capabilities,
-                            f"lookup {self.resource_name} with external_id {missing}",
-                        )
-                # Raise the original error if it's not a 400 or the user has access to read the resource.from
-                raise
-            self._cache.update(lookup)
-            self._reverse_cache.update({v: k for k, v in lookup.items()})
-            if len(missing) != len(lookup) and not is_dry_run:
-                raise ResourceRetrievalError(
-                    f"Failed to retrieve {self.resource_name} with external_id {missing}. Have you created it?", missing
-                )
-        return (
-            self._get_id_from_cache(external_id, is_dry_run, allow_empty)
-            if isinstance(external_id, str)
-            else [self._get_id_from_cache(id, is_dry_run, allow_empty) for id in ids]
-        )
+            need_lookup.remove("")
+        if need_lookup:
+            self._do_lookup_external_ids(need_lookup, is_dry_run)
 
-    def _get_id_from_cache(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int:
+        if isinstance(external_id, str):
+            return self._get_id_from_cache(external_id, is_dry_run, allow_empty)
+        else:
+            internal_ids = (
+                self._get_id_from_cache(external_id, is_dry_run, allow_empty) for external_id in external_id
+            )
+            return [id_ for id_ in internal_ids if id_ is not None]
+
+    def _do_lookup_external_ids(self, external_ids: list[str], is_dry_run: bool) -> None:
+        try:
+            ids_by_external_id = self._id(external_ids)
+        except CogniteAPIError as e:
+            if 400 <= e.code < 500:
+                missing_capabilities = self._toolkit_client.verify.authorization(self._read_acl())
+                if missing_capabilities:
+                    raise self._toolkit_client.verify.create_error(
+                        missing_capabilities,
+                        f"lookup {self.resource_name} with external_id {external_ids}",
+                    )
+            # Raise the original error if it's not a 400 or the user has access to read the resource.from
+            raise
+        self._cache.update(ids_by_external_id)
+        self._reverse_cache.update({v: k for k, v in ids_by_external_id.items()})
+        missing_external_ids = [ext_id for ext_id in external_ids if ext_id not in ids_by_external_id]
+        if missing_external_ids and not is_dry_run:
+            plural = "s" if len(missing_external_ids) > 1 else ""
+            plural2 = "They do" if len(missing_external_ids) > 1 else "It does"
+            MediumSeverityWarning(
+                f"Failed to retrieve {self.resource_name} with external_id{plural} "
+                f"{humanize_collection(missing_external_ids)}. {plural2} not exist in CDF"
+            ).print_warning(console=self._console)
+            self._cache.update({ext_id: None for ext_id in missing_external_ids})
+
+    def _get_id_from_cache(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int | None:
         if allow_empty and external_id == "":
             return 0
         elif is_dry_run:
@@ -96,34 +122,56 @@ class LookUpAPI(ToolkitAPI, ABC):
     def external_id(self, id: Sequence[int]) -> list[str]: ...
 
     def external_id(self, id: int | Sequence[int]) -> str | None | list[str]:
+        """Lookup external IDs for given internal IDs.
+
+        Args:
+            id: An integer ID or a sequence of integer IDs to look up. Note that an ID of 0 corresponds
+                to an empty string external ID.
+
+        Returns:
+            The corresponding external ID as a string if a single ID is provided,
+            or a list of external IDs if a sequence of IDs is provided.
+            If an ID does not exist, None is returned for that ID.
+
+        """
         ids = [id] if isinstance(id, int) else id
-        missing = [id_ for id_ in ids if id_ not in self._reverse_cache]
-        if 0 in missing:
-            missing.remove(0)
-        if missing:
-            try:
-                lookup = self._external_id(missing)
-            except CogniteAPIError as e:
-                if 400 <= e.code < 500:
-                    missing_capabilities = self._toolkit_client.verify.authorization(self._read_acl())
-                    if missing_capabilities:
-                        raise self._toolkit_client.verify.create_error(
-                            missing_capabilities,
-                            f"lookup {self.resource_name} with id {missing}",
-                        )
-                # Raise the original error if it's not a 400 or the user has access to read the resource.from
-                raise
-            self._reverse_cache.update(lookup)
-            self._cache.update({v: k for k, v in lookup.items()})
-            if len(missing) != len(lookup):
-                MediumSeverityWarning(
-                    f"Failed to retrieve {self.resource_name} with id {missing}. It does not exist in CDF"
-                ).print_warning()
+        need_lookup = [id_ for id_ in ids if id_ not in self._reverse_cache if id_ != 0]
+        if need_lookup:
+            self._do_lookup_internal_ids(need_lookup)
+
         if isinstance(id, int):
             return self._get_external_id_from_cache(id)
         else:
-            external_ids = (self._get_external_id_from_cache(id) for id in ids)
-            return [id for id in external_ids if id is not None]
+            external_ids = (self._get_external_id_from_cache(id_) for id_ in ids)
+            return [id_ for id_ in external_ids if id_ is not None]
+
+    def _do_lookup_internal_ids(self, ids: list[int]) -> None:
+        try:
+            found_by_id = self._external_id(ids)
+        except CogniteAPIError as e:
+            if 400 <= e.code < 500:
+                missing_capabilities = self._toolkit_client.verify.authorization(self._read_acl())
+                if missing_capabilities:
+                    raise self._toolkit_client.verify.create_error(
+                        missing_capabilities,
+                        f"lookup {self.resource_name} with id {ids}",
+                    )
+            # Raise the original error if it's not a 400 or the user has access to read the resource.from
+            raise
+        self._reverse_cache.update(found_by_id)
+        self._cache.update({v: k for k, v in found_by_id.items()})
+        missing_ids = [id for id in ids if id not in found_by_id]
+        if not missing_ids:
+            return None
+        plural = "s" if len(missing_ids) > 1 else ""
+        plural2 = "They do" if len(missing_ids) > 1 else "It does"
+        MediumSeverityWarning(
+            f"Failed to retrieve {self.resource_name} with id{plural} "
+            f"{humanize_collection(missing_ids)}. {plural2} not exist in CDF"
+        ).print_warning(console=self._console)
+        # Cache the missing IDs with None to avoid repeated lookups
+        self._reverse_cache.update({missing_id: None for missing_id in missing_ids})
+        return None
 
     def _get_external_id_from_cache(self, id: int) -> str | None:
         if id == 0:
@@ -311,8 +359,10 @@ class FunctionLookUpAPI(LookUpAPI):
 
 
 class AllLookUpAPI(LookUpAPI, ABC):
-    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: "ToolkitClient") -> None:
-        super().__init__(config, api_version, cognite_client)
+    def __init__(
+        self, config: ClientConfig, api_version: str | None, cognite_client: "ToolkitClient", console: Console
+    ) -> None:
+        super().__init__(config, api_version, cognite_client, console)
         self._has_looked_up = False
 
     @abstractmethod
@@ -322,13 +372,15 @@ class AllLookUpAPI(LookUpAPI, ABC):
     def _id(self, external_id: SequenceNotStr[str]) -> dict[str, int]:
         if not self._has_looked_up:
             self._lookup()
-        return {external_id: self._cache[external_id] for external_id in external_id if external_id in self._cache}
+        found_pairs = ((ext_id, self._cache[ext_id]) for ext_id in external_id if ext_id in self._cache)
+        return {k: v for k, v in found_pairs if v is not None}
 
     def _external_id(self, id: Sequence[int]) -> dict[int, str]:
         if not self._has_looked_up:
             self._lookup()
             self._has_looked_up = True
-        return {id: self._reverse_cache[id] for id in id}
+        found_pairs = ((id_, self._reverse_cache[id_]) for id_ in id if id_ in self._reverse_cache)
+        return {k: v for k, v in found_pairs if v is not None}
 
 
 class SecurityCategoriesLookUpAPI(AllLookUpAPI):
@@ -363,18 +415,21 @@ class LocationFiltersLookUpAPI(AllLookUpAPI):
     def _id(self, external_id: SequenceNotStr[str]) -> dict[str, int]:
         if not self._has_looked_up:
             self._lookup()
-        return {external_id: self._cache[external_id] for external_id in external_id if external_id in self._cache}
+        found_pairs = ((ext_id, self._cache[ext_id]) for ext_id in external_id if ext_id in self._cache)
+        return {k: v for k, v in found_pairs if v is not None}
 
 
 class LookUpGroup(ToolkitAPI):
-    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: "ToolkitClient") -> None:
+    def __init__(
+        self, config: ClientConfig, api_version: str | None, cognite_client: "ToolkitClient", console: Console
+    ) -> None:
         super().__init__(config, api_version, cognite_client)
-        self.data_sets = DataSetLookUpAPI(config, api_version, cognite_client)
-        self.assets = AssetLookUpAPI(config, api_version, cognite_client)
-        self.time_series = TimeSeriesLookUpAPI(config, api_version, cognite_client)
-        self.files = FileMetadataLookUpAPI(config, api_version, cognite_client)
-        self.events = EventLookUpAPI(config, api_version, cognite_client)
-        self.security_categories = SecurityCategoriesLookUpAPI(config, api_version, cognite_client)
-        self.location_filters = LocationFiltersLookUpAPI(config, api_version, cognite_client)
-        self.extraction_pipelines = ExtractionPipelineLookUpAPI(config, api_version, cognite_client)
-        self.functions = FunctionLookUpAPI(config, api_version, cognite_client)
+        self.data_sets = DataSetLookUpAPI(config, api_version, cognite_client, console)
+        self.assets = AssetLookUpAPI(config, api_version, cognite_client, console)
+        self.time_series = TimeSeriesLookUpAPI(config, api_version, cognite_client, console)
+        self.files = FileMetadataLookUpAPI(config, api_version, cognite_client, console)
+        self.events = EventLookUpAPI(config, api_version, cognite_client, console)
+        self.security_categories = SecurityCategoriesLookUpAPI(config, api_version, cognite_client, console)
+        self.location_filters = LocationFiltersLookUpAPI(config, api_version, cognite_client, console)
+        self.extraction_pipelines = ExtractionPipelineLookUpAPI(config, api_version, cognite_client, console)
+        self.functions = FunctionLookUpAPI(config, api_version, cognite_client, console)

cognite_toolkit/_cdf_tk/client/testing.py

@@ -9,6 +9,7 @@ from cognite.client._api.functions import FunctionCallsAPI, FunctionSchedulesAPI
 from cognite.client._api.raw import RawDatabasesAPI, RawRowsAPI, RawTablesAPI
 from cognite.client._api.synthetic_time_series import SyntheticDatapointsAPI
 from cognite.client.testing import CogniteClientMock
+from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
 
@@ -58,6 +59,7 @@ class ToolkitClientMock(CogniteClientMock):
             super().__init__(*args, **kwargs)
             return None
         super().__init__(*args, **kwargs)
+        self.console = Console()
         # Developer note:
         # - Please add your mocked APIs in chronological order
         # - For nested APIs:

cognite_toolkit/_cdf_tk/commands/_download.py

@@ -3,7 +3,6 @@ from functools import partial
 from pathlib import Path
 
 from cognite.client.data_classes._base import T_CogniteResource
-from rich.console import Console
 
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
@@ -41,7 +40,7 @@ class DownloadCommand(ToolkitCommand):
         """
         compression_cls = Compression.from_name(compression)
 
-        console = Console()
+        console = io.client.console
         for selector in selectors:
             target_dir = output_dir / selector.group
             if verbose:

cognite_toolkit/_cdf_tk/commands/_migrate/creators.py

@@ -22,7 +22,7 @@ from cognite.client.data_classes.events import EventProperty
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.apm_config_v1 import APMConfig, APMConfigList
 from cognite_toolkit._cdf_tk.cruds import NodeCRUD, ResourceCRUD, SpaceCRUD
-from cognite_toolkit._cdf_tk.exceptions import ToolkitRequiredValueError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError, ToolkitRequiredValueError
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 
 from .data_model import CREATED_SOURCE_SYSTEM_VIEW_ID, SPACE, SPACE_SOURCE_VIEW_ID
@@ -196,9 +196,10 @@ class SourceSystemCreator(MigrationCreator[NodeApplyList]):
     @cached_property
     def _advanced_filter(self) -> filters.Filter:
         if self.data_set_external_id is not None:
-            return filters.Equals(
-                SourceFileProperty.data_set_id, self.client.lookup.data_sets.id(self.data_set_external_id)
-            )
+            data_set_id = self.client.lookup.data_sets.id(self.data_set_external_id)
+            if data_set_id is None:
+                raise ToolkitMissingResourceError(f"Data set with external ID '{self.data_set_external_id}' not found.")
+            return filters.Equals(SourceFileProperty.data_set_id, data_set_id)
         if self.hierarchy is not None:
             return filters.InAssetSubtree(SourceFileProperty.asset_external_ids, [self.hierarchy])
         else:

cognite_toolkit/_cdf_tk/commands/_purge.py

@@ -375,6 +375,8 @@ class PurgeCommand(ToolkitCommand):
         # Validate Auth
         validator = ValidateAccess(client, "purge")
         data_set_id = client.lookup.data_sets.id(selected_data_set_external_id)
+        if data_set_id is None:
+            raise ToolkitMissingResourceError(f"DataSet {selected_data_set_external_id!r} does not exist")
         action = cast(Sequence[Literal["read", "write"]], ["read"] if dry_run else ["read", "write"])
         if include_data:
             # Check asset, events, time series, files, and sequences access, relationships, labels, 3D access.

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py

@@ -60,7 +60,7 @@ class _ReplaceMethod:
     """This is a small helper class used in the
     lookup and replace in the ACL scoped ids"""
 
-    lookup_method: Callable[[str, bool], int]
+    lookup_method: Callable[[str, bool], int | None]
     reverse_lookup_method: Callable[[int], str | None]
     id_name: str
 

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py

@@ -36,7 +36,6 @@ from cognite_toolkit._cdf_tk.client.data_classes.functions import FunctionSchedu
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
     ResourceCreationError,
-    ResourceRetrievalError,
     ToolkitRequiredValueError,
 )
 from cognite_toolkit._cdf_tk.feature_flags import Flags
@@ -565,22 +564,7 @@ class FunctionScheduleCRUD(
 
     def create(self, items: FunctionScheduleWriteList) -> FunctionSchedulesList:
         created_list = FunctionSchedulesList([], cognite_client=self.client)
-        functions_to_lookup = list({item.function_external_id for item in items if item.function_external_id})
-        function_id_by_external_id: dict[str, int] = {}
-        if functions_to_lookup:
-            try:
-                function_ids = self.client.lookup.functions.id(functions_to_lookup)
-            except ResourceRetrievalError as e:
-                failed_items = self.get_ids(items)
-                missing_functions = functions_to_lookup
-                if e.resources:
-                    missing_functions = list(e.resources)
-                    failed_items = [id_ for id_ in failed_items if id_.function_external_id in set(missing_functions)]
-                raise ResourceCreationError(
-                    f"Failed to create function schedule(s) {humanize_collection(failed_items)}. "
-                    f"Could not find function(s) {humanize_collection(missing_functions)!r}"
-                ) from e
-            function_id_by_external_id = dict(zip(functions_to_lookup, function_ids))
+        function_id_by_external_id = self._get_function_ids_by_external_id(items)
 
         for item in items:
             id_ = self.get_id(item)
@@ -607,6 +591,28 @@ class FunctionScheduleCRUD(
             created_list.append(created)
         return created_list
 
+    def _get_function_ids_by_external_id(self, items: FunctionScheduleWriteList) -> dict[str, int]:
+        functions_to_lookup = list({item.function_external_id for item in items if item.function_external_id})
+        if not functions_to_lookup:
+            return {}
+        function_ids = self.client.lookup.functions.id(functions_to_lookup)
+
+        if len(function_ids) == len(functions_to_lookup):
+            return dict(zip(functions_to_lookup, function_ids))
+        # The lookup API is cached, so it is cheap to do individual lookups to find the missing ones.
+        lookup_pair = (
+            (function_external_id, self.client.lookup.functions.id(function_external_id))
+            for function_external_id in functions_to_lookup
+        )
+        missing_functions = {func for func, func_id in lookup_pair if func_id is None}
+        failed_schedules = [self.get_id(item) for item in items if item.function_external_id in missing_functions]
+        plural_schedules = "s" if len(failed_schedules) > 1 else ""
+        plural_fun = "s" if len(missing_functions) > 1 else ""
+        raise ResourceCreationError(
+            f"Failed to create function schedule{plural_schedules} {humanize_collection(failed_schedules)}. "
+            f"Could not find function{plural_fun} {humanize_collection(missing_functions)!r}"
+        )
+
     def delete(self, ids: SequenceNotStr[FunctionScheduleID]) -> int:
         schedules = self.retrieve(ids)
         count = 0

cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -36,10 +36,9 @@ from cognite_toolkit._cdf_tk.cruds import (
     EventCRUD,
     FileMetadataCRUD,
     LabelCRUD,
-    ResourceCRUD,
     TimeSeriesCRUD,
 )
-from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError, ToolkitNotImplementedError
 from cognite_toolkit._cdf_tk.utils.aggregators import (
     AssetAggregator,
     AssetCentricAggregator,
@@ -85,19 +84,10 @@ class BaseAssetCentricIO(
 
     def __init__(self, client: ToolkitClient) -> None:
         super().__init__(client)
-        self._loader = self._get_loader()
         self._aggregator = self._get_aggregator()
         self._downloaded_data_sets_by_selector: dict[AssetCentricSelector, set[int]] = defaultdict(set)
         self._downloaded_labels_by_selector: dict[AssetCentricSelector, set[str]] = defaultdict(set)
 
-    @abstractmethod
-    def _get_loader(
-        self,
-    ) -> ResourceCRUD[
-        T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-    ]:
-        raise NotImplementedError()
-
     @abstractmethod
     def _get_aggregator(self) -> AssetCentricAggregator:
         raise NotImplementedError()
@@ -113,11 +103,6 @@ class BaseAssetCentricIO(
             return self._aggregator.count(hierarchy=selector.hierarchy)
         return None
 
-    def data_to_json_chunk(
-        self, data_chunk: Sequence[T_WritableCogniteResource], selector: AssetCentricSelector | None = None
-    ) -> list[dict[str, JsonVal]]:
-        return [self._loader.dump_resource(item) for item in data_chunk]
-
     def configurations(self, selector: AssetCentricSelector) -> Iterable[StorageIOConfig]:
         data_set_ids = self._downloaded_data_sets_by_selector[selector]
         if data_set_ids:
@@ -178,6 +163,22 @@ class BaseAssetCentricIO(
     def create_internal_identifier(cls, internal_id: int, project: str) -> str:
         return f"INTERNAL_ID_project_{project}_{internal_id!s}"
 
+    def _populate_data_set_cache(self, chunk: Sequence[Asset | FileMetadata | TimeSeries | Event]) -> None:
+        data_set_ids = {item.data_set_id for item in chunk if item.data_set_id is not None}
+        self.client.lookup.data_sets.external_id(list(data_set_ids))
+
+    def _populate_security_category_cache(self, chunk: Sequence[FileMetadata | TimeSeries]) -> None:
+        security_category_ids: set[int] = set()
+        for item in chunk:
+            security_category_ids.update(item.security_categories or [])
+        self.client.lookup.security_categories.external_id(list(security_category_ids))
+
+    def _populate_asset_cache(self, chunk: Sequence[FileMetadata | Event]) -> None:
+        asset_ids: set[int] = set()
+        for item in chunk:
+            asset_ids.update(item.asset_ids or [])
+        self.client.lookup.assets.external_id(list(asset_ids))
+
 
 class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetList]):
     KIND = "Assets"
@@ -187,22 +188,31 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
     SUPPORTED_READ_FORMATS = frozenset({".parquet", ".csv", ".ndjson", ".yaml", ".yml"})
     UPLOAD_ENDPOINT = "/assets"
 
+    def __init__(self, client: ToolkitClient) -> None:
+        super().__init__(client)
+        self._crud = AssetCRUD.create_loader(self.client)
+
     def as_id(self, item: Asset) -> str:
         return item.external_id if item.external_id is not None else self._create_identifier(item.id)
 
-    def _get_loader(self) -> AssetCRUD:
-        return AssetCRUD.create_loader(self.client)
-
     def _get_aggregator(self) -> AssetCentricAggregator:
         return AssetAggregator(self.client)
 
     def get_schema(self, selector: AssetCentricSelector) -> list[SchemaColumn]:
         data_set_ids: list[int] = []
         if isinstance(selector, DataSetSelector):
-            data_set_ids.append(self.client.lookup.data_sets.id(selector.data_set_external_id))
+            data_set_id = self.client.lookup.data_sets.id(selector.data_set_external_id)
+            if data_set_id is None:
+                raise ToolkitMissingResourceError(
+                    f"Data set with external ID {selector.data_set_external_id} not found."
+                )
+            data_set_ids.append(data_set_id)
         hierarchy: list[int] = []
         if isinstance(selector, AssetSubtreeSelector):
-            hierarchy.append(self.client.lookup.assets.id(selector.hierarchy))
+            asset_id = self.client.lookup.assets.id(selector.hierarchy)
+            if asset_id is None:
+                raise ToolkitMissingResourceError(f"Asset with external ID {selector.hierarchy} not found.")
+            hierarchy.append(asset_id)
 
         if hierarchy or data_set_ids:
             metadata_keys = metadata_key_counts(
@@ -238,8 +248,17 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
             self._collect_dependencies(asset_list, selector)
             yield Page(worker_id="main", items=asset_list)
 
+    def data_to_json_chunk(
+        self, data_chunk: Sequence[Asset], selector: AssetCentricSelector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        # Ensure data sets are looked up to populate cache.
+        # This is to avoid looking up each data set id individually in the .dump_resource call.
+        self._populate_data_set_cache(data_chunk)
+
+        return [self._crud.dump_resource(item) for item in data_chunk]
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> AssetWrite:
-        return self._loader.load_resource(item_json)
+        return self._crud.load_resource(item_json)
 
     def retrieve(self, ids: Sequence[int]) -> AssetList:
         return self.client.assets.retrieve_multiple(ids)
@@ -253,19 +272,25 @@ class FileMetadataIO(BaseAssetCentricIO[str, FileMetadataWrite, FileMetadata, Fi
     SUPPORTED_READ_FORMATS = frozenset({".parquet", ".csv", ".ndjson"})
     UPLOAD_ENDPOINT = "/files"
 
+    def __init__(self, client: ToolkitClient) -> None:
+        super().__init__(client)
+        self._crud = FileMetadataCRUD.create_loader(self.client)
+
     def as_id(self, item: FileMetadata) -> str:
         return item.external_id if item.external_id is not None else self._create_identifier(item.id)
 
-    def _get_loader(self) -> FileMetadataCRUD:
-        return FileMetadataCRUD.create_loader(self.client)
-
     def _get_aggregator(self) -> AssetCentricAggregator:
         return FileAggregator(self.client)
 
     def get_schema(self, selector: AssetCentricSelector) -> list[SchemaColumn]:
         data_set_ids: list[int] = []
         if isinstance(selector, DataSetSelector):
-            data_set_ids.append(self.client.lookup.data_sets.id(selector.data_set_external_id))
+            data_set_id = self.client.lookup.data_sets.id(selector.data_set_external_id)
+            if data_set_id is None:
+                raise ToolkitMissingResourceError(
+                    f"Data set with external ID {selector.data_set_external_id} not found."
+                )
+            data_set_ids.append(data_set_id)
         if isinstance(selector, AssetSubtreeSelector):
             raise ToolkitNotImplementedError(f"Selector type {type(selector)} not supported for FileIO.")
 
@@ -345,8 +370,19 @@ class FileMetadataIO(BaseAssetCentricIO[str, FileMetadataWrite, FileMetadata, Fi
     def retrieve(self, ids: Sequence[int]) -> FileMetadataList:
         return self.client.files.retrieve_multiple(ids)
 
+    def data_to_json_chunk(
+        self, data_chunk: Sequence[FileMetadata], selector: AssetCentricSelector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        # Ensure data sets/assets/security-categories are looked up to populate cache.
+        # This is to avoid looking up each data set id individually in the .dump_resource call
+        self._populate_data_set_cache(data_chunk)
+        self._populate_asset_cache(data_chunk)
+        self._populate_security_category_cache(data_chunk)
+
+        return [self._crud.dump_resource(item) for item in data_chunk]
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> FileMetadataWrite:
-        return self._loader.load_resource(item_json)
+        return self._crud.load_resource(item_json)
 
 
 class TimeSeriesIO(BaseAssetCentricIO[str, TimeSeriesWrite, TimeSeries, TimeSeriesWriteList, TimeSeriesList]):
@@ -357,12 +393,13 @@ class TimeSeriesIO(BaseAssetCentricIO[str, TimeSeriesWrite, TimeSeries, TimeSeri
     UPLOAD_ENDPOINT = "/timeseries"
     RESOURCE_TYPE = "timeseries"
 
+    def __init__(self, client: ToolkitClient) -> None:
+        super().__init__(client)
+        self._crud = TimeSeriesCRUD.create_loader(self.client)
+
     def as_id(self, item: TimeSeries) -> str:
         return item.external_id if item.external_id is not None else self._create_identifier(item.id)
 
-    def _get_loader(self) -> TimeSeriesCRUD:
-        return TimeSeriesCRUD.create_loader(self.client)
-
     def _get_aggregator(self) -> AssetCentricAggregator:
         return TimeSeriesAggregator(self.client)
 
@@ -380,13 +417,29 @@ class TimeSeriesIO(BaseAssetCentricIO[str, TimeSeriesWrite, TimeSeries, TimeSeri
             self._collect_dependencies(ts_list, selector)
             yield Page(worker_id="main", items=ts_list)
 
+    def data_to_json_chunk(
+        self, data_chunk: Sequence[TimeSeries], selector: AssetCentricSelector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        # Ensure data sets/assets/security categories are looked up to populate cache.
+        self._populate_data_set_cache(data_chunk)
+        self._populate_security_category_cache(data_chunk)
+        asset_ids = {item.asset_id for item in data_chunk if item.asset_id is not None}
+        self.client.lookup.assets.external_id(list(asset_ids))
+
+        return [self._crud.dump_resource(item) for item in data_chunk]
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> TimeSeriesWrite:
-        return self._loader.load_resource(item_json)
+        return self._crud.load_resource(item_json)
 
     def get_schema(self, selector: AssetCentricSelector) -> list[SchemaColumn]:
         data_set_ids: list[int] = []
         if isinstance(selector, DataSetSelector):
-            data_set_ids.append(self.client.lookup.data_sets.id(selector.data_set_external_id))
+            data_set_id = self.client.lookup.data_sets.id(selector.data_set_external_id)
+            if data_set_id is None:
+                raise ToolkitMissingResourceError(
+                    f"Data set with external ID {selector.data_set_external_id} not found."
+                )
+            data_set_ids.append(data_set_id)
         elif isinstance(selector, AssetSubtreeSelector):
             raise ToolkitNotImplementedError(f"Selector type {type(selector)} not supported for {type(self).__name__}.")
 
@@ -424,19 +477,25 @@ class EventIO(BaseAssetCentricIO[str, EventWrite, Event, EventWriteList, EventLi
     UPLOAD_ENDPOINT = "/events"
     RESOURCE_TYPE = "event"
 
+    def __init__(self, client: ToolkitClient) -> None:
+        super().__init__(client)
+        self._crud = EventCRUD.create_loader(self.client)
+
     def as_id(self, item: Event) -> str:
         return item.external_id if item.external_id is not None else self._create_identifier(item.id)
 
-    def _get_loader(self) -> EventCRUD:
-        return EventCRUD.create_loader(self.client)
-
     def _get_aggregator(self) -> AssetCentricAggregator:
         return EventAggregator(self.client)
 
     def get_schema(self, selector: AssetCentricSelector) -> list[SchemaColumn]:
         data_set_ids: list[int] = []
         if isinstance(selector, DataSetSelector):
-            data_set_ids.append(self.client.lookup.data_sets.id(selector.data_set_external_id))
+            data_set_id = self.client.lookup.data_sets.id(selector.data_set_external_id)
+            if data_set_id is None:
+                raise ToolkitMissingResourceError(
+                    f"Data set with external ID {selector.data_set_external_id} not found."
+                )
+            data_set_ids.append(data_set_id)
         hierarchy: list[int] = []
         if isinstance(selector, AssetSubtreeSelector):
             raise ToolkitNotImplementedError(f"Selector type {type(selector)} not supported for {type(self).__name__}.")
@@ -476,8 +535,17 @@ class EventIO(BaseAssetCentricIO[str, EventWrite, Event, EventWriteList, EventLi
             self._collect_dependencies(event_list, selector)
             yield Page(worker_id="main", items=event_list)
 
+    def data_to_json_chunk(
+        self, data_chunk: Sequence[Event], selector: AssetCentricSelector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        # Ensure data sets/assets are looked up to populate cache.
+        self._populate_data_set_cache(data_chunk)
+        self._populate_asset_cache(data_chunk)
+
+        return [self._crud.dump_resource(item) for item in data_chunk]
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> EventWrite:
-        return self._loader.load_resource(item_json)
+        return self._crud.load_resource(item_json)
 
     def retrieve(self, ids: Sequence[int]) -> EventList:
         return self.client.events.retrieve_multiple(ids)

cognite_toolkit/_cdf_tk/utils/aggregators.py

@@ -18,6 +18,7 @@ from cognite.client.data_classes.sequences import SequenceProperty
 from cognite.client.data_classes.time_series import TimeSeriesProperty
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError
 from cognite_toolkit._cdf_tk.utils.cdf import (
     label_aggregate_count,
     label_count,
@@ -125,7 +126,9 @@ class MetadataAggregator(AssetCentricAggregator, ABC, Generic[T_CogniteFilter]):
         self, hierarchy: str | list[str] | None = None, data_sets: str | list[str] | None = None
     ) -> list[tuple[str, int]]:
         """Returns a list of metadata keys and their counts."""
-        hierarchy_ids, data_set_ids = self._lookup_hierarchy_data_set_pair(hierarchy, data_sets)
+        hierarchy_ids, data_set_ids = self._lookup_hierarchy_data_set_pair(
+            hierarchy, data_sets, operation="find metadata keys"
+        )
         return self._used_metadata_keys(hierarchy=hierarchy_ids, data_sets=data_set_ids)
 
     @lru_cache(maxsize=1)
@@ -140,22 +143,44 @@ class MetadataAggregator(AssetCentricAggregator, ABC, Generic[T_CogniteFilter]):
         )
 
     def _lookup_hierarchy_data_set_pair(
-        self, hierarchy: str | list[str] | None = None, data_sets: str | list[str] | None = None
+        self, hierarchy: str | list[str] | None, data_sets: str | list[str] | None, operation: str
     ) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]:
         """Returns a tuple of hierarchy and data sets."""
         hierarchy_ids: tuple[int, ...] | None = None
         if isinstance(hierarchy, str):
             asset_id = self.client.lookup.assets.id(external_id=hierarchy, allow_empty=False)
+            if asset_id is None:
+                raise ToolkitMissingResourceError(f"Cannot {operation}. Asset with external ID {hierarchy!r} not found")
             hierarchy_ids = (asset_id,)
         elif isinstance(hierarchy, list) and all(isinstance(item, str) for item in hierarchy):
-            hierarchy_ids = tuple(sorted(self.client.lookup.assets.id(external_id=hierarchy, allow_empty=False)))
+            asset_ids = self.client.lookup.assets.id(external_id=hierarchy, allow_empty=False)
+            if len(asset_ids) != len(hierarchy):
+                missing = set(hierarchy) - set(
+                    self.client.lookup.assets.external_id([id_ for id_ in asset_ids if id_ is not None])
+                )
+                raise ToolkitMissingResourceError(
+                    f"Cannot {operation}. Assets with external IDs {sorted(missing)!r} not found"
+                )
+            hierarchy_ids = tuple(sorted(asset_ids))
 
         data_set_ids: tuple[int, ...] | None = None
         if isinstance(data_sets, str):
             data_set_id = self.client.lookup.data_sets.id(external_id=data_sets, allow_empty=False)
+            if data_set_id is None:
+                raise ToolkitMissingResourceError(
+                    f"Cannot {operation}. Data set with external ID {data_sets!r} not found"
+                )
             data_set_ids = (data_set_id,)
         elif isinstance(data_sets, list) and all(isinstance(item, str) for item in data_sets):
-            data_set_ids = tuple(sorted(self.client.lookup.data_sets.id(external_id=data_sets, allow_empty=False)))
+            data_set_ids_list = self.client.lookup.data_sets.id(external_id=data_sets, allow_empty=False)
+            if len(data_set_ids_list) != len(data_sets):
+                missing = set(data_sets) - set(
+                    self.client.lookup.data_sets.external_id([id_ for id_ in data_set_ids_list if id_ is not None])
+                )
+                raise ToolkitMissingResourceError(
+                    f"Cannot {operation}. Data sets with external IDs {sorted(missing)!r} not found"
+                )
+            data_set_ids = tuple(sorted(data_set_ids_list))
 
         return hierarchy_ids, data_set_ids
 
@@ -201,7 +226,9 @@ class LabelAggregator(MetadataAggregator, ABC, Generic[T_CogniteFilter]):
         self, hierarchy: str | list[str] | None = None, data_sets: str | list[str] | None = None
     ) -> list[tuple[str, int]]:
         """Returns a list of labels and their counts."""
-        hierarchy_ids, data_set_ids = self._lookup_hierarchy_data_set_pair(hierarchy, data_sets)
+        hierarchy_ids, data_set_ids = self._lookup_hierarchy_data_set_pair(
+            hierarchy, data_sets, operation="find labels"
+        )
         return self._used_labels(hierarchy=hierarchy_ids, data_sets=data_set_ids)
 
     @lru_cache(maxsize=1)

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.83
+      image: cognite/toolkit:0.6.84
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.83
+      image: cognite/toolkit:0.6.84
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.83"
+version = "0.6.84"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.83"
+__version__ = "0.6.84"

{cognite_toolkit-0.6.83.dist-info → cognite_toolkit-0.6.84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.83
+Version: 0.6.84
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.83.dist-info → cognite_toolkit-0.6.84.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=1OSAvbOeuIrnsczEG2BtGqRP3L3sq0VMPthmugnqCUw,5821
-cognite_toolkit/_version.py,sha256=NoDb44OMue8eXw7UG1FCeT7NcrsMhObnFzSRH_iUix8,23
+cognite_toolkit/_version.py,sha256=VnWi3C5S4335obqqoSD8NeZPSw1Kg8vPGHxnwrQk6dU,23
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=DAUmHf19ByVIGH4MDPdXKHZ0G97CxdD5J-EzHTq66C8,8025
 cognite_toolkit/_cdf_tk/constants.py,sha256=e9XmGvQCqGq7zYQrNoopU5e2KnYZYBPyUC5raGShK7k,6364
@@ -35,10 +35,10 @@ cognite_toolkit/_cdf_tk/builders/_streamlit.py,sha256=8Pu_zgyKZjbAsPWywjzB2KWD7h
 cognite_toolkit/_cdf_tk/builders/_transformation.py,sha256=STB42zhzOW5M_-b8cKOQ_cegnr7FtMoMxZ87gPLXft4,4723
 cognite_toolkit/_cdf_tk/client/__init__.py,sha256=a6rQXDGfW2g7K5WwrOW5oakh1TdFlBjUVjf9wusOox8,135
 cognite_toolkit/_cdf_tk/client/_constants.py,sha256=COUGcea37mDF2sf6MGqJXWmecTY_6aCImslxXrYW1I0,73
-cognite_toolkit/_cdf_tk/client/_toolkit_client.py,sha256=TNQNYKfVSTeBbaGJOKnD5fc_U5QiWdb574QrAvkZd5g,2542
+cognite_toolkit/_cdf_tk/client/_toolkit_client.py,sha256=lwVpUOTfkN2dx-0rKYIVi7BNPF17lMpf-LEaEdnS4ZM,2622
 cognite_toolkit/_cdf_tk/client/api_client.py,sha256=CQdD_gfDqQkz5OYHrTnKvBvEvzHPdHDB1BkZPWRoahg,440
 cognite_toolkit/_cdf_tk/client/config.py,sha256=weMR43z-gqHMn-Jqvfmh_nJ0HbgEdyeCGtISuEf3OuY,4269
-cognite_toolkit/_cdf_tk/client/testing.py,sha256=77QQd6vroe7LUOY9T1s6qW261XUjkrilL9AYM8pG_YY,5989
+cognite_toolkit/_cdf_tk/client/testing.py,sha256=lpcxC2l6Dj8foZr4IboqbIzVxR85NI6G_5saH2DHSfg,6055
 cognite_toolkit/_cdf_tk/client/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/client/api/canvas.py,sha256=i2NwyhvmklTPx3e-yd4lvSxyn6JEjSpv8WXa1SxtmV8,8789
 cognite_toolkit/_cdf_tk/client/api/charts.py,sha256=t-VOrRGwpjmYUtUqGObQWYwGb5gOHVp4cHZBm8ZVGn0,4953
@@ -50,7 +50,7 @@ cognite_toolkit/_cdf_tk/client/api/extended_raw.py,sha256=9DVbM2aWmIyzbaW-lh10_p
 cognite_toolkit/_cdf_tk/client/api/extended_timeseries.py,sha256=xK7XhTfe4W9FvaueUIfR7Q64JOIDwq_svHRjORM76Q4,17774
 cognite_toolkit/_cdf_tk/client/api/fixed_transformations.py,sha256=m66cqbx4oCtjv5TBQOWLNFrz475qVTCXBu_pTxbdCD4,5589
 cognite_toolkit/_cdf_tk/client/api/location_filters.py,sha256=TIbomUbpUNDxOON_a3pwBmCBdltxL1jMQBXKcIjRx44,3759
-cognite_toolkit/_cdf_tk/client/api/lookup.py,sha256=QDMGuNmbvd7NUjUlfjp_v2KWoXcx0lw75WZXxgVC9iA,14974
+cognite_toolkit/_cdf_tk/client/api/lookup.py,sha256=c-cvtgfGGGYyk8ROcJu44qlo1ocqbk0o1zafCql79fU,17652
 cognite_toolkit/_cdf_tk/client/api/migration.py,sha256=eIKcUAXHblL0vL9UzyZgifnL4bW2N4TUt613hTTFf1w,15156
 cognite_toolkit/_cdf_tk/client/api/project.py,sha256=Hj0uDCLyPofG-T4626EdeoRRtBaovhU-SMAQ7VWJ1M4,1063
 cognite_toolkit/_cdf_tk/client/api/search.py,sha256=L4cDPip7pJVP7bEgAiSOjqINIHg8AULNBtR29G5khEQ,612
@@ -95,9 +95,9 @@ cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=OJYtHiERtUBXm3cjUTyPVaYIMVQp
 cognite_toolkit/_cdf_tk/commands/_base.py,sha256=m2hnXo_AAHhsoSayHZO_zUa4xEt5w5oMB4WCHmJr-AY,2595
 cognite_toolkit/_cdf_tk/commands/_changes.py,sha256=DIwuiRpDhWBDpsW3R3yqj0eWLAE3c_kPbmCaUkxjFuo,24852
 cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMoulWgJzbDC8YIIQDPJ5x3w,1011
-cognite_toolkit/_cdf_tk/commands/_download.py,sha256=S5WuOAgbZTgU5kXyjqgTzm7e0hN9FI6IYOeQ9U-44Ow,6718
+cognite_toolkit/_cdf_tk/commands/_download.py,sha256=qkbzHzd6FZydNiG83vwciBJEmQAG0t9EFfvZb0K89TA,6693
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
-cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=aZ9c7iVrKlvgX62ZbYRKlRqx5eThNu6lheaUIFQ_FOo,41281
+cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=7meU96Xni0TezAZz14wB310bcf6Ps6KRHFOJOvnIlsQ,41420
 cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=kXYmP1YMg-JvsuN1iYaMuZH1qZfapya2j-RABGhqGHo,11860
 cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=ARlbqA_5ZWlgN3-xF-zanzSx4B0-9ULnguA5QgHmKGA,1225
 cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
@@ -119,7 +119,7 @@ cognite_toolkit/_cdf_tk/commands/_migrate/base.py,sha256=aS32Wa-gd7vFNOdCTKEIpSS
 cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py,sha256=Tv4OG9V6tDsQbSH13YW8M0n8Ury5gU16oJB-OISVR0w,6398
 cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=RtC5CMQC1Wy5RmOwlCM67Mc8fZEExXubJLvzDzk2IMA,14097
 cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=Eol-0ruQ14fwS-bx2pEmbXdICodfknSJ_OsAASa6jkY,9592
-cognite_toolkit/_cdf_tk/commands/_migrate/creators.py,sha256=GMKwpJWH-yQmS9QUFcjKklI_9Ynxwp58R-vZlzJOBuI,9411
+cognite_toolkit/_cdf_tk/commands/_migrate/creators.py,sha256=FTu7w3G8KyPY8pagG3KdPpOmpLcjehaAg2auEy6iM7A,9605
 cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=eF3Fv-tm_A8iaRUAOVOdg1DKPN7OYVPwrLQMvawsO14,8572
 cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=L9HJGmcTiAkC8Sftrngp8FyHBtAbkZtb9HrpLFg-ED0,6006
 cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=i1eUsNX6Dueol9STIEwyksBnBsWUk13O8qHIjW964pM,7860
@@ -134,7 +134,7 @@ cognite_toolkit/_cdf_tk/cruds/_data_cruds.py,sha256=3FYKow5uOBWt2-6kzszRJE8YgqxJ
 cognite_toolkit/_cdf_tk/cruds/_worker.py,sha256=-jbl4JV18Ow3y8BcecYPx8XxDIP897A80yD7frwuilc,9369
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py,sha256=IE6WxGh9AthWxwHnDmj8EJDd7q6W2OGf1vMnr_lrKAY,2769
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py,sha256=2pcpwAJQ4GPvW2jM0J-Odm7d1sTlaaoBLKEXPHHz2VQ,5091
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py,sha256=s-oDIYu27zhsMRLKKIHJj862fQOMHwVVKoxh1I8jfIE,24964
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py,sha256=iGG2_btpEqip3o6OKpcKfrh5IljOH9NbrJcGBKX0bn4,24971
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py,sha256=JIY5qC2mg3kV5zIMZRvUuTi3z7NM7gGbd3eiTN8nI8o,25716
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py,sha256=KrL7bj8q5q18mGB2V-NDkW5U5nfseZOyorXiUbp2uLw,6100
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py,sha256=iXn9iAtwA8mhH-7j9GF-MlLomTcaw3GhEbFY28Wx0iA,9927
@@ -142,7 +142,7 @@ cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=KED-wNXTZbkrK4
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py,sha256=G27TZF1aTN3aruJ1HTfpvhI4fZyHkn-uD6NJpKgkSow,18605
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py,sha256=Gtcu-i-nejPCN0Uxn8O5_QakdX2wgDcVCJn1X7AMu-I,11638
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py,sha256=F3n2FOWAPder4z3OTYs81VB-6C6r3oUzJsHvigdhaD0,15500
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py,sha256=JSxkDw9yqdv_jGS_tS_BuqkpSvBSFTkirxp6CMX6TRc,28777
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py,sha256=3qWJPy5syAAILaQRnsn_EfLyL23w3E6WLLdrrvUj7KI,29158
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/group_scoped.py,sha256=WEg8-CxMP64WfE_XXIlH114zM51K0uLaYa4atd992zI,1690
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/hosted_extractors.py,sha256=7y2ffuLjUAafGIXfZqqRkXopQKemmFr_IPi_lD4k-fo,15434
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/industrial_tool.py,sha256=x_hHlFZ1AURmixRKltWQ680JgrEr6CswMFyaG3N-fnk,8011
@@ -227,7 +227,7 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Q
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
 cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=aM-skaPnKTH1B7HG0faeTUNf7u1b-sT8l7hh5JRZ1E8,2288
 cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=bhyG1d2_9duPkX-otC2brVcpChvdXSPkYhBHS5T_72g,4343
-cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=0Q8htH7VM2mw0xx64R6lYrOLuLW8OJ4awO1kQgblh9E,23390
+cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=spFAdoHxGKDDEWItM4RovSddudwoPIa_uev8WCAQ9zs,27027
 cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=yqAaBlZcE53V_HKcKi_q-OjpC2Nnhkw13cbSvXjx2wY,8682
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=_tKOdlo7tMJoh7y-47o7sySfDMRa-G-AFVprmzjn3EQ,9311
@@ -245,7 +245,7 @@ cognite_toolkit/_cdf_tk/tk_warnings/fileread.py,sha256=d2Kx6YyLmCkyFNjK8MO6eKGce
 cognite_toolkit/_cdf_tk/tk_warnings/other.py,sha256=4T-WRqMG-KKFS0AV3w6ilea22G_MTXFcOKpbd2z6MNE,5624
 cognite_toolkit/_cdf_tk/utils/__init__.py,sha256=-X01eYNwz3l0W2jby0DZzlDIe9HEhUUj-dK8DBhYki8,1413
 cognite_toolkit/_cdf_tk/utils/_auxiliary.py,sha256=tvvgFiWwLOCVDkPg83U5XqBLfOOt_gI3697EQr7-GSE,1198
-cognite_toolkit/_cdf_tk/utils/aggregators.py,sha256=WU0E3YpDm2e2p7OTrIZ2eY8e9-CFaGpcL7HbAbzqj-8,15947
+cognite_toolkit/_cdf_tk/utils/aggregators.py,sha256=8rgNNgrH1VhynFBzIe-bkwz1TU3DZSDJqwb6agxXzzs,17357
 cognite_toolkit/_cdf_tk/utils/auth.py,sha256=O_VuLXS8xYdjBeTATIHWht7KtemmhZIfxUrzgqCdrMQ,23106
 cognite_toolkit/_cdf_tk/utils/auxiliary.py,sha256=BLhWEBz8ErY4CVwJOaN6H9pUrkMJBT-ylYNuvDORMXQ,814
 cognite_toolkit/_cdf_tk/utils/cdf.py,sha256=hxLBhen-ad60FsGk26b1fp-dMper6Ue-qmD643A79i4,18282
@@ -284,13 +284,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=OZbAHAwQZdUoUp4l5yVtSjif-TDdKdqrRpaYs6Cno3o,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=tDMSTeVoP8gVemK0DQpmYzWC7D-jSgY3o4BVZNC-NII,2430
-cognite_toolkit/_resources/cdf.toml,sha256=lp87nN7ZxGnPiocONpDzYsAeeOz3K-nec3Dd5ZR1u9Q,487
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=CEJyS9A4yJ3QyTKT0-bQn_gAej6TFxSyKAeh7U-x_GE,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=JETNf3gik_QNu5EC5UfGt7YSxUIpqG_g-xqb8QqReak,2430
+cognite_toolkit/_resources/cdf.toml,sha256=iMFIlLuzS4GaaMM0lBi7Ve-Pt8Ce4kbYKqqvVb5GtEg,487
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.83.dist-info/METADATA,sha256=hw2mPua88MAh5KVj90XnAU9e-QA0OnU9PGrmzZKmTSY,4501
-cognite_toolkit-0.6.83.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.83.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.83.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.83.dist-info/RECORD,,
+cognite_toolkit-0.6.84.dist-info/METADATA,sha256=dnNPS0fE8GblOccgwHL6srQ8B7DmdquASwQSbZxsoK4,4501
+cognite_toolkit-0.6.84.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.84.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.84.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.84.dist-info/RECORD,,