cognite-toolkit 0.6.92__py3-none-any.whl → 0.6.93__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -9,7 +9,14 @@ from rich.console import Console
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
-from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, are_same_kind, get_upload_io
+from cognite_toolkit._cdf_tk.storageio import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    T_Selector,
+    UploadableStorageIO,
+    are_same_kind,
+    get_upload_io,
+)
 from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
 from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
@@ -182,11 +189,18 @@ class UploadCommand(ToolkitCommand):
                     is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
                     if is_table and not isinstance(io, TableUploadableStorageIO):
                         raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+                    if isinstance(io, AssetIO):
+                        # Assets needs to be uploaded from the root asset to the leaves. The specialized reader
+                        # handles this by iterating over the file n times, where n is the depth of the asset hierarchy.
+                        reader = AssetFileReaderAdapter(reader)
                     tracker = ProgressTracker[str]([self._UPLOAD])
                     data_name = "row" if is_table else "line"
                     executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
                         download_iterable=chunker(
-                            ((f"{data_name} {line_no}", item) for line_no, item in enumerate(reader.read_chunks(), 1)),
+                            (
+                                (f"{data_name} {line_no}", item)
+                                for line_no, item in reader.read_chunks_with_line_numbers()
+                            ),
                             io.CHUNK_SIZE,
                         ),
                         process=partial(io.rows_to_data, selector=selector)

cognite_toolkit/_cdf_tk/storageio/__init__.py

@@ -5,7 +5,15 @@ from cognite_toolkit._cdf_tk.utils.fileio import COMPRESSION_BY_SUFFIX
 
 from ._annotations import FileAnnotationIO
 from ._applications import CanvasIO, ChartIO
-from ._asset_centric import AssetIO, BaseAssetCentricIO, EventIO, FileMetadataIO, HierarchyIO, TimeSeriesIO
+from ._asset_centric import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    BaseAssetCentricIO,
+    EventIO,
+    FileMetadataIO,
+    HierarchyIO,
+    TimeSeriesIO,
+)
 from ._base import (
     ConfigurableStorageIO,
     Page,
@@ -45,6 +53,7 @@ def are_same_kind(kind: str, kind_or_path: str | Path, /) -> bool:
 
 
 __all__ = [
+    "AssetFileReaderAdapter",
     "AssetIO",
     "BaseAssetCentricIO",
     "CanvasIO",

cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from collections.abc import Iterable, MutableSequence, Sequence
+from collections.abc import Iterable, Iterator, MutableSequence, Sequence
+from io import TextIOWrapper
 from typing import ClassVar, Generic
 
 from cognite.client.data_classes import (
@@ -48,7 +49,7 @@ from cognite_toolkit._cdf_tk.utils.aggregators import (
     TimeSeriesAggregator,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import metadata_key_counts
-from cognite_toolkit._cdf_tk.utils.fileio import SchemaColumn
+from cognite_toolkit._cdf_tk.utils.fileio import FileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.http_client import (
     FailedRequestItems,
     FailedRequestMessage,
@@ -639,3 +640,44 @@ class HierarchyIO(ConfigurableStorageIO[AssetCentricSelector, AssetCentricResour
 
     def get_resource_io(self, kind: str) -> BaseAssetCentricIO:
         return self._io_by_kind[kind]
+
+
+class AssetFileReaderAdapter(FileReader):
+    """Adapter of the FileReader to read asset-centric data.
+
+    This is used when uploading assets from files to account for the hierarchical structure. It returns the assets
+    by the depth in the hierarchy, starting from the root assets and going down to the leaf assets.
+
+    Args:
+        other_reader (FileReader): The underlying FileReader to read data from.
+    """
+
+    def __init__(self, other_reader: FileReader) -> None:
+        super().__init__(other_reader.input_file)
+        self._other_reader = other_reader
+
+    def read_chunks(self) -> Iterator[dict[str, JsonVal]]:
+        """Reads chunks from the file, yielding each chunk, sorted by asset depth."""
+        yield from (item for _, item in self.read_chunks_with_line_numbers())
+
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        current_depth = max_depth = 0
+        while current_depth <= max_depth:
+            for line_number, item in self._other_reader.read_chunks_with_line_numbers():
+                try:
+                    depth = int(item.get("depth"))  # type: ignore[arg-type]
+                except (TypeError, ValueError):
+                    if current_depth == 0:
+                        # If depth is not set, we yield it at depth 0
+                        yield line_number, item
+                    continue
+                if depth == current_depth:
+                    yield line_number, item
+                elif current_depth == 0:
+                    max_depth = max(max_depth, depth)
+            current_depth += 1
+
+    def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
+        # This method is not used by AssetFileReaderAdapter as read_chunks is overridden.
+        # It is implemented to satisfy the abstract base class, but should not be called.
+        raise NotImplementedError(f"{type(self).__name__} does not implement '_read_chunks_from_file'.")

cognite_toolkit/_cdf_tk/utils/fileio/_readers.py

@@ -30,6 +30,10 @@ class FileReader(FileIO, ABC):
         with compression.open("r") as file:
             yield from self._read_chunks_from_file(file)
 
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        """Read chunks from the file, yielding each chunk with its corresponding line number."""
+        yield from enumerate(self.read_chunks(), start=1)
+
     @abstractmethod
     def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
         """Read chunks from the file."""

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.92
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.92
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.92"
+version = "0.6.93"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.92"
+__version__ = "0.6.93"

{cognite_toolkit-0.6.92.dist-info → cognite_toolkit-0.6.93.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.92
+Version: 0.6.93
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.92.dist-info → cognite_toolkit-0.6.93.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=1OSAvbOeuIrnsczEG2BtGqRP3L3sq0VMPthmugnqCUw,5821
-cognite_toolkit/_version.py,sha256=YZ7t39hsVsYWfIrytObsq9bNDMC8cNjBCtEcAsIIKtA,23
+cognite_toolkit/_version.py,sha256=mhDTOECXK-iUdY0_1-p2mAdb1w2FkaLj3EepXd5qWHI,23
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=IjmzNVLxsOV6tsMDgmJmXsy-LQru-8IEQdFzGW5DxVk,8117
 cognite_toolkit/_cdf_tk/constants.py,sha256=e9XmGvQCqGq7zYQrNoopU5e2KnYZYBPyUC5raGShK7k,6364
@@ -104,7 +104,7 @@ cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMou
 cognite_toolkit/_cdf_tk/commands/_download.py,sha256=OBKPM_HGGA1i32th1SAgkQM_81CUFvm39kGqBuOeeTs,6816
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
 cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=RadQHsmkPez3fZ5HCP9b82o2_fBx8P_-bTo7prkvWXU,32525
-cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=Y0k0q4Iu4F7g3Ax3slSrpll3AHxmODYNq55waHw4mzc,12473
+cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=iJ5GILjYJfU_vVu-ie8CExxF0rKaZPN0sQ-JcC42JW4,12977
 cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=UxMJW5QYKts4om5n6x2Tq2ihvfO9gWjhQKeqZNFTlKg,402
 cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
 cognite_toolkit/_cdf_tk/commands/auth.py,sha256=N6JgtF0_Qoh-xM8VlBb_IK1n0Lo5I7bIkIHmXm1l7ug,31638
@@ -232,10 +232,10 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/data_postprocessing.py,sha256=
 cognite_toolkit/_cdf_tk/resource_classes/robotics/frame.py,sha256=XmDqJ0pAxe_vAP0Dhktba1f9o2zg_ORCJ3Hz8cyJMrk,899
 cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Qt15hHoR63SM7pg321BhNuTNjI7HHCwSA,468
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
-cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=-OhPPhl9z1ynYcFlRVYfOPD246HhQydmn4VzByJh7C0,2355
+cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=Vx9E1glHeQJ8NzL30uphBvgs-toTjzeBNU72ewr0WHU,2442
 cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=wAMkgM-IpgXuY7_1KbtiTv8VdA555ywKjntD_boOBPk,4647
 cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=bhyG1d2_9duPkX-otC2brVcpChvdXSPkYhBHS5T_72g,4343
-cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=taxMxR2uBsCUsaproukSZxYEtly2sy2aZqPhI_-zhPA,28896
+cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=heh1rlrVnBhnfR_6DijdotoJC7l6_jhAGHhV5-X3nAc,30927
 cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=NWXPdgzUnpBiav5Hi8XGHkWU9QiMjNzBQTxMcuxF-LA,11017
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=_tKOdlo7tMJoh7y-47o7sySfDMRa-G-AFVprmzjn3EQ,9311
@@ -280,7 +280,7 @@ cognite_toolkit/_cdf_tk/utils/validate_access.py,sha256=1puswcpgEDNCwdk91dhLqCBS
 cognite_toolkit/_cdf_tk/utils/fileio/__init__.py,sha256=ts5kYu_1Ks7xjnM6pIrVUrZe0nkYI6euYXeE4ox34xk,1199
 cognite_toolkit/_cdf_tk/utils/fileio/_base.py,sha256=MpWaD3lR9vrJ-kGzTiDOtChXhvFD7-xrP-Pzp7vjnLY,756
 cognite_toolkit/_cdf_tk/utils/fileio/_compression.py,sha256=8BAPgg5OKc3vkEEkqOvYsuyh12iXVNuEmC0omWwyJNQ,2355
-cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=nGfsSfpXDYUncncsFuJD9-xYPJ5635mSFUJfuCyQ3no,13724
+cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=mBf0-8JFwLfyTGJH8nWpbn89VPTj9UwP3GmZtx8t3A4,13969
 cognite_toolkit/_cdf_tk/utils/fileio/_writers.py,sha256=ghNGBZjkISAlbxe8o5YWWloLXG9QKOtF_qGA9JkvYss,17712
 cognite_toolkit/_cdf_tk/utils/http_client/__init__.py,sha256=H1T-cyIoVaPL4MvN1IuG-cHgj-cqB7eszu2kIN939lw,813
 cognite_toolkit/_cdf_tk/utils/http_client/_client.py,sha256=sN1Sizxv_rAcOSRd-2lKSs0p-SO1rA4eHL8pHMTBx54,11018
@@ -292,13 +292,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=wEKwRNF7C9cq34yIaQjhQcIANNqWKVXwgESz25Mh8gs,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=mV6glTAupf2E-Cci5C56kOG6_bGjGzX29hEpcAX1pJo,2430
-cognite_toolkit/_resources/cdf.toml,sha256=lvpWokDF_cDC9TVVer2bgMao9P--Hv0oJSpHl65OQ0Q,487
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=gGC7ISonjAKg43Y3dqlpKBrIgr8zpx8Qqy8Q1sWvyV8,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=GscUUdSipJkZhn3Q-sxqjm8aB7p65Mty10ZEjYBXgCY,2430
+cognite_toolkit/_resources/cdf.toml,sha256=mXqc38IbnyCiBG9DXm7RHnIWxjFoEdqOy1h0oEiB-hk,487
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.92.dist-info/METADATA,sha256=urDOVGuS1ZjJwkVHotxDnAmMSLASTfKRwuy_1ymxEu0,4501
-cognite_toolkit-0.6.92.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.92.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.92.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.92.dist-info/RECORD,,
+cognite_toolkit-0.6.93.dist-info/METADATA,sha256=VXtoYqT3_e-jS49YiM4DwRVAW_2mqtJPGqwFskF6Ai0,4501
+cognite_toolkit-0.6.93.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.93.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.93.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.93.dist-info/RECORD,,