cognite-toolkit 0.6.91__py3-none-any.whl → 0.6.93__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -9,7 +9,14 @@ from rich.console import Console
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
-from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, are_same_kind, get_upload_io
+from cognite_toolkit._cdf_tk.storageio import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    T_Selector,
+    UploadableStorageIO,
+    are_same_kind,
+    get_upload_io,
+)
 from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
 from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
@@ -182,11 +189,18 @@ class UploadCommand(ToolkitCommand):
                     is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
                     if is_table and not isinstance(io, TableUploadableStorageIO):
                         raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+                    if isinstance(io, AssetIO):
+                        # Assets needs to be uploaded from the root asset to the leaves. The specialized reader
+                        # handles this by iterating over the file n times, where n is the depth of the asset hierarchy.
+                        reader = AssetFileReaderAdapter(reader)
                     tracker = ProgressTracker[str]([self._UPLOAD])
                     data_name = "row" if is_table else "line"
                     executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
                         download_iterable=chunker(
-                            ((f"{data_name} {line_no}", item) for line_no, item in enumerate(reader.read_chunks(), 1)),
+                            (
+                                (f"{data_name} {line_no}", item)
+                                for line_no, item in reader.read_chunks_with_line_numbers()
+                            ),
                             io.CHUNK_SIZE,
                         ),
                         process=partial(io.rows_to_data, selector=selector)

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py

@@ -6,6 +6,7 @@ from typing import Any, final
 
 import pandas as pd
 from cognite.client.data_classes import (
+    AggregateResultItem,
     Asset,
     AssetList,
     AssetWrite,
@@ -213,6 +214,20 @@ class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset, AssetWriteList, AssetList])
             dumped.pop("metadata", None)
         if "parentId" in dumped and "parentId" not in local:
             dumped.pop("parentId")
+        if resource.aggregates:
+            # This is only included when the asset is downloaded/migrated with aggregated properties
+            aggregates = (
+                resource.aggregates.dump()
+                if isinstance(resource.aggregates, AggregateResultItem)
+                else resource.aggregates
+            )
+            if "path" in aggregates:
+                path = aggregates.pop("path", [])
+                if path:
+                    aggregates["path"] = self.client.lookup.assets.external_id(
+                        [segment["id"] for segment in path if "id" in segment]
+                    )
+            dumped.update(aggregates)
         return dumped
 
 

cognite_toolkit/_cdf_tk/storageio/__init__.py

@@ -5,7 +5,15 @@ from cognite_toolkit._cdf_tk.utils.fileio import COMPRESSION_BY_SUFFIX
 
 from ._annotations import FileAnnotationIO
 from ._applications import CanvasIO, ChartIO
-from ._asset_centric import AssetIO, BaseAssetCentricIO, EventIO, FileMetadataIO, HierarchyIO, TimeSeriesIO
+from ._asset_centric import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    BaseAssetCentricIO,
+    EventIO,
+    FileMetadataIO,
+    HierarchyIO,
+    TimeSeriesIO,
+)
 from ._base import (
     ConfigurableStorageIO,
     Page,
@@ -45,6 +53,7 @@ def are_same_kind(kind: str, kind_or_path: str | Path, /) -> bool:
 
 
 __all__ = [
+    "AssetFileReaderAdapter",
     "AssetIO",
     "BaseAssetCentricIO",
     "CanvasIO",

cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -1,9 +1,11 @@
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from collections.abc import Iterable, MutableSequence, Sequence
+from collections.abc import Iterable, Iterator, MutableSequence, Sequence
+from io import TextIOWrapper
 from typing import ClassVar, Generic
 
 from cognite.client.data_classes import (
+    AggregateResultItem,
     Asset,
     AssetList,
     AssetWrite,
@@ -47,7 +49,7 @@ from cognite_toolkit._cdf_tk.utils.aggregators import (
     TimeSeriesAggregator,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import metadata_key_counts
-from cognite_toolkit._cdf_tk.utils.fileio import SchemaColumn
+from cognite_toolkit._cdf_tk.utils.fileio import FileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.http_client import (
     FailedRequestItems,
     FailedRequestMessage,
@@ -267,6 +269,9 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
             SchemaColumn(name="source", type="string"),
             SchemaColumn(name="labels", type="string", is_array=True),
             SchemaColumn(name="geoLocation", type="json"),
+            SchemaColumn(name="childCount", type="integer"),
+            SchemaColumn(name="depth", type="integer"),
+            SchemaColumn(name="path", type="string", is_array=True),
         ]
         return asset_schema + metadata_schema
 
@@ -277,6 +282,7 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
             limit=limit,
             asset_subtree_external_ids=asset_subtree_external_ids,
             data_set_external_ids=data_set_external_ids,
+            aggregated_properties=["child_count", "path", "depth"],
         ):
             self._collect_dependencies(asset_list, selector)
             yield Page(worker_id="main", items=asset_list)
@@ -287,6 +293,14 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
         # Ensure data sets are looked up to populate cache.
         # This is to avoid looking up each data set id individually in the .dump_resource call.
         self._populate_data_set_cache(data_chunk)
+        asset_ids = {
+            segment["id"]
+            for item in data_chunk
+            if isinstance(item.aggregates, AggregateResultItem)
+            for segment in item.aggregates.path or []
+            if "id" in segment
+        }
+        self.client.lookup.assets.external_id(list(asset_ids))
 
         return [self._crud.dump_resource(item) for item in data_chunk]
 
@@ -626,3 +640,44 @@ class HierarchyIO(ConfigurableStorageIO[AssetCentricSelector, AssetCentricResour
 
     def get_resource_io(self, kind: str) -> BaseAssetCentricIO:
         return self._io_by_kind[kind]
+
+
+class AssetFileReaderAdapter(FileReader):
+    """Adapter of the FileReader to read asset-centric data.
+
+    This is used when uploading assets from files to account for the hierarchical structure. It returns the assets
+    by the depth in the hierarchy, starting from the root assets and going down to the leaf assets.
+
+    Args:
+        other_reader (FileReader): The underlying FileReader to read data from.
+    """
+
+    def __init__(self, other_reader: FileReader) -> None:
+        super().__init__(other_reader.input_file)
+        self._other_reader = other_reader
+
+    def read_chunks(self) -> Iterator[dict[str, JsonVal]]:
+        """Reads chunks from the file, yielding each chunk, sorted by asset depth."""
+        yield from (item for _, item in self.read_chunks_with_line_numbers())
+
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        current_depth = max_depth = 0
+        while current_depth <= max_depth:
+            for line_number, item in self._other_reader.read_chunks_with_line_numbers():
+                try:
+                    depth = int(item.get("depth"))  # type: ignore[arg-type]
+                except (TypeError, ValueError):
+                    if current_depth == 0:
+                        # If depth is not set, we yield it at depth 0
+                        yield line_number, item
+                    continue
+                if depth == current_depth:
+                    yield line_number, item
+                elif current_depth == 0:
+                    max_depth = max(max_depth, depth)
+            current_depth += 1
+
+    def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
+        # This method is not used by AssetFileReaderAdapter as read_chunks is overridden.
+        # It is implemented to satisfy the abstract base class, but should not be called.
+        raise NotImplementedError(f"{type(self).__name__} does not implement '_read_chunks_from_file'.")

cognite_toolkit/_cdf_tk/utils/fileio/_readers.py

@@ -30,6 +30,10 @@ class FileReader(FileIO, ABC):
         with compression.open("r") as file:
             yield from self._read_chunks_from_file(file)
 
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        """Read chunks from the file, yielding each chunk with its corresponding line number."""
+        yield from enumerate(self.read_chunks(), start=1)
+
     @abstractmethod
     def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
         """Read chunks from the file."""

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.91
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.91
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.91"
+version = "0.6.93"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.91"
+__version__ = "0.6.93"

{cognite_toolkit-0.6.91.dist-info → cognite_toolkit-0.6.93.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.91
+Version: 0.6.93
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.91.dist-info → cognite_toolkit-0.6.93.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=1OSAvbOeuIrnsczEG2BtGqRP3L3sq0VMPthmugnqCUw,5821
-cognite_toolkit/_version.py,sha256=9Ri169KKHyKxaZSidD_j90UcNV5T3e6F_wIJBgWZ0sY,23
+cognite_toolkit/_version.py,sha256=mhDTOECXK-iUdY0_1-p2mAdb1w2FkaLj3EepXd5qWHI,23
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=IjmzNVLxsOV6tsMDgmJmXsy-LQru-8IEQdFzGW5DxVk,8117
 cognite_toolkit/_cdf_tk/constants.py,sha256=e9XmGvQCqGq7zYQrNoopU5e2KnYZYBPyUC5raGShK7k,6364
@@ -104,7 +104,7 @@ cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMou
 cognite_toolkit/_cdf_tk/commands/_download.py,sha256=OBKPM_HGGA1i32th1SAgkQM_81CUFvm39kGqBuOeeTs,6816
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
 cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=RadQHsmkPez3fZ5HCP9b82o2_fBx8P_-bTo7prkvWXU,32525
-cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=Y0k0q4Iu4F7g3Ax3slSrpll3AHxmODYNq55waHw4mzc,12473
+cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=iJ5GILjYJfU_vVu-ie8CExxF0rKaZPN0sQ-JcC42JW4,12977
 cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=UxMJW5QYKts4om5n6x2Tq2ihvfO9gWjhQKeqZNFTlKg,402
 cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
 cognite_toolkit/_cdf_tk/commands/auth.py,sha256=N6JgtF0_Qoh-xM8VlBb_IK1n0Lo5I7bIkIHmXm1l7ug,31638
@@ -140,7 +140,7 @@ cognite_toolkit/_cdf_tk/cruds/_worker.py,sha256=XdLm6DMFln9DqDgEbeqzepw9WRSXx7Wd
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py,sha256=ArtGm8mpf_akIAGMY_E6fyGpKQrMVJMkRZ3GEwCl8ig,2829
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py,sha256=2pcpwAJQ4GPvW2jM0J-Odm7d1sTlaaoBLKEXPHHz2VQ,5091
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py,sha256=iGG2_btpEqip3o6OKpcKfrh5IljOH9NbrJcGBKX0bn4,24971
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py,sha256=JIY5qC2mg3kV5zIMZRvUuTi3z7NM7gGbd3eiTN8nI8o,25716
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py,sha256=7RdiWvh6MLI1lLmt3gcqDQj61xbwREhsvoyjFuJn2F0,26402
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py,sha256=KrL7bj8q5q18mGB2V-NDkW5U5nfseZOyorXiUbp2uLw,6100
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py,sha256=iXn9iAtwA8mhH-7j9GF-MlLomTcaw3GhEbFY28Wx0iA,9927
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=KED-wNXTZbkrK4bNMsJPnP1d0j8hF5apDUPgaKj52TY,57130
@@ -232,10 +232,10 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/data_postprocessing.py,sha256=
 cognite_toolkit/_cdf_tk/resource_classes/robotics/frame.py,sha256=XmDqJ0pAxe_vAP0Dhktba1f9o2zg_ORCJ3Hz8cyJMrk,899
 cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Qt15hHoR63SM7pg321BhNuTNjI7HHCwSA,468
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
-cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=-OhPPhl9z1ynYcFlRVYfOPD246HhQydmn4VzByJh7C0,2355
+cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=Vx9E1glHeQJ8NzL30uphBvgs-toTjzeBNU72ewr0WHU,2442
 cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=wAMkgM-IpgXuY7_1KbtiTv8VdA555ywKjntD_boOBPk,4647
 cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=bhyG1d2_9duPkX-otC2brVcpChvdXSPkYhBHS5T_72g,4343
-cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=Rhy64zUW4oxacq_vYomDeTRPmF6Vx-1mkYAFAqJE9vk,28312
+cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=heh1rlrVnBhnfR_6DijdotoJC7l6_jhAGHhV5-X3nAc,30927
 cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=NWXPdgzUnpBiav5Hi8XGHkWU9QiMjNzBQTxMcuxF-LA,11017
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=_tKOdlo7tMJoh7y-47o7sySfDMRa-G-AFVprmzjn3EQ,9311
@@ -280,7 +280,7 @@ cognite_toolkit/_cdf_tk/utils/validate_access.py,sha256=1puswcpgEDNCwdk91dhLqCBS
 cognite_toolkit/_cdf_tk/utils/fileio/__init__.py,sha256=ts5kYu_1Ks7xjnM6pIrVUrZe0nkYI6euYXeE4ox34xk,1199
 cognite_toolkit/_cdf_tk/utils/fileio/_base.py,sha256=MpWaD3lR9vrJ-kGzTiDOtChXhvFD7-xrP-Pzp7vjnLY,756
 cognite_toolkit/_cdf_tk/utils/fileio/_compression.py,sha256=8BAPgg5OKc3vkEEkqOvYsuyh12iXVNuEmC0omWwyJNQ,2355
-cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=nGfsSfpXDYUncncsFuJD9-xYPJ5635mSFUJfuCyQ3no,13724
+cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=mBf0-8JFwLfyTGJH8nWpbn89VPTj9UwP3GmZtx8t3A4,13969
 cognite_toolkit/_cdf_tk/utils/fileio/_writers.py,sha256=ghNGBZjkISAlbxe8o5YWWloLXG9QKOtF_qGA9JkvYss,17712
 cognite_toolkit/_cdf_tk/utils/http_client/__init__.py,sha256=H1T-cyIoVaPL4MvN1IuG-cHgj-cqB7eszu2kIN939lw,813
 cognite_toolkit/_cdf_tk/utils/http_client/_client.py,sha256=sN1Sizxv_rAcOSRd-2lKSs0p-SO1rA4eHL8pHMTBx54,11018
@@ -292,13 +292,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=b7kF2xhXglUAQbNX0QfAgjhzKgOcubmPTebEW2wxMtU,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=unYeq3F9O7KP6zW-P1XIwZMkXyXjyC_M7RMbA6VsR2w,2430
-cognite_toolkit/_resources/cdf.toml,sha256=1Nh-OE694YugqEwAeCM2CqDAvzxfnKen4vfmmPqbJCk,487
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=gGC7ISonjAKg43Y3dqlpKBrIgr8zpx8Qqy8Q1sWvyV8,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=GscUUdSipJkZhn3Q-sxqjm8aB7p65Mty10ZEjYBXgCY,2430
+cognite_toolkit/_resources/cdf.toml,sha256=mXqc38IbnyCiBG9DXm7RHnIWxjFoEdqOy1h0oEiB-hk,487
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.91.dist-info/METADATA,sha256=6Iti9CJTHQCwpGNr4zID5Y1uHhtVaj8RJ4ePoF0F-V8,4501
-cognite_toolkit-0.6.91.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.91.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.91.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.91.dist-info/RECORD,,
+cognite_toolkit-0.6.93.dist-info/METADATA,sha256=VXtoYqT3_e-jS49YiM4DwRVAW_2mqtJPGqwFskF6Ai0,4501
+cognite_toolkit-0.6.93.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.93.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.93.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.93.dist-info/RECORD,,