cognite-toolkit 0.6.92__tar.gz → 0.6.93__tar.gz

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.92
+Version: 0.6.93
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cdf.toml

@@ -31,7 +31,7 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.92"
+version = "0.6.93"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -9,7 +9,14 @@ from rich.console import Console
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
-from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, are_same_kind, get_upload_io
+from cognite_toolkit._cdf_tk.storageio import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    T_Selector,
+    UploadableStorageIO,
+    are_same_kind,
+    get_upload_io,
+)
 from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
 from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
@@ -182,11 +189,18 @@ class UploadCommand(ToolkitCommand):
                     is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
                     if is_table and not isinstance(io, TableUploadableStorageIO):
                         raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+                    if isinstance(io, AssetIO):
+                        # Assets needs to be uploaded from the root asset to the leaves. The specialized reader
+                        # handles this by iterating over the file n times, where n is the depth of the asset hierarchy.
+                        reader = AssetFileReaderAdapter(reader)
                     tracker = ProgressTracker[str]([self._UPLOAD])
                     data_name = "row" if is_table else "line"
                     executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
                         download_iterable=chunker(
-                            ((f"{data_name} {line_no}", item) for line_no, item in enumerate(reader.read_chunks(), 1)),
+                            (
+                                (f"{data_name} {line_no}", item)
+                                for line_no, item in reader.read_chunks_with_line_numbers()
+                            ),
                             io.CHUNK_SIZE,
                         ),
                         process=partial(io.rows_to_data, selector=selector)

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/storageio/__init__.py

@@ -5,7 +5,15 @@ from cognite_toolkit._cdf_tk.utils.fileio import COMPRESSION_BY_SUFFIX
 
 from ._annotations import FileAnnotationIO
 from ._applications import CanvasIO, ChartIO
-from ._asset_centric import AssetIO, BaseAssetCentricIO, EventIO, FileMetadataIO, HierarchyIO, TimeSeriesIO
+from ._asset_centric import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    BaseAssetCentricIO,
+    EventIO,
+    FileMetadataIO,
+    HierarchyIO,
+    TimeSeriesIO,
+)
 from ._base import (
     ConfigurableStorageIO,
     Page,
@@ -45,6 +53,7 @@ def are_same_kind(kind: str, kind_or_path: str | Path, /) -> bool:
 
 
 __all__ = [
+    "AssetFileReaderAdapter",
     "AssetIO",
     "BaseAssetCentricIO",
     "CanvasIO",

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from collections.abc import Iterable, MutableSequence, Sequence
+from collections.abc import Iterable, Iterator, MutableSequence, Sequence
+from io import TextIOWrapper
 from typing import ClassVar, Generic
 
 from cognite.client.data_classes import (
@@ -48,7 +49,7 @@ from cognite_toolkit._cdf_tk.utils.aggregators import (
     TimeSeriesAggregator,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import metadata_key_counts
-from cognite_toolkit._cdf_tk.utils.fileio import SchemaColumn
+from cognite_toolkit._cdf_tk.utils.fileio import FileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.http_client import (
     FailedRequestItems,
     FailedRequestMessage,
@@ -639,3 +640,44 @@ class HierarchyIO(ConfigurableStorageIO[AssetCentricSelector, AssetCentricResour
 
     def get_resource_io(self, kind: str) -> BaseAssetCentricIO:
         return self._io_by_kind[kind]
+
+
+class AssetFileReaderAdapter(FileReader):
+    """Adapter of the FileReader to read asset-centric data.
+
+    This is used when uploading assets from files to account for the hierarchical structure. It returns the assets
+    by the depth in the hierarchy, starting from the root assets and going down to the leaf assets.
+
+    Args:
+        other_reader (FileReader): The underlying FileReader to read data from.
+    """
+
+    def __init__(self, other_reader: FileReader) -> None:
+        super().__init__(other_reader.input_file)
+        self._other_reader = other_reader
+
+    def read_chunks(self) -> Iterator[dict[str, JsonVal]]:
+        """Reads chunks from the file, yielding each chunk, sorted by asset depth."""
+        yield from (item for _, item in self.read_chunks_with_line_numbers())
+
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        current_depth = max_depth = 0
+        while current_depth <= max_depth:
+            for line_number, item in self._other_reader.read_chunks_with_line_numbers():
+                try:
+                    depth = int(item.get("depth"))  # type: ignore[arg-type]
+                except (TypeError, ValueError):
+                    if current_depth == 0:
+                        # If depth is not set, we yield it at depth 0
+                        yield line_number, item
+                    continue
+                if depth == current_depth:
+                    yield line_number, item
+                elif current_depth == 0:
+                    max_depth = max(max_depth, depth)
+            current_depth += 1
+
+    def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
+        # This method is not used by AssetFileReaderAdapter as read_chunks is overridden.
+        # It is implemented to satisfy the abstract base class, but should not be called.
+        raise NotImplementedError(f"{type(self).__name__} does not implement '_read_chunks_from_file'.")

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/utils/fileio/_readers.py

@@ -30,6 +30,10 @@ class FileReader(FileIO, ABC):
         with compression.open("r") as file:
             yield from self._read_chunks_from_file(file)
 
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        """Read chunks from the file, yielding each chunk with its corresponding line number."""
+        yield from enumerate(self.read_chunks(), start=1)
+
     @abstractmethod
     def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
         """Read chunks from the file."""

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.92
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.92
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.92"
+version = "0.6.93"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.93/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.93"

cognite_toolkit-0.6.93/last_changelog_entry.md

@@ -0,0 +1,11 @@
+## cdf 
+
+### Improved
+
+- [alpha] The `cdf data upload` command can now upload an asset
+hierarchy. It does this by first uploading root assets, then one level
+deep, two level deep, and so on.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.93/last_git_message.txt

@@ -0,0 +1,27 @@
+[CDF-26220, CDF-26218] 🤖 Upload assets hierarchy (#2180)
+
+# Description
+
+Handling uploading of asset hierarchy by iterating over the file
+multiple times. Note that the alternative would be to do a sort in
+memory, but the data plugin is optimized for not exploding in memory
+thus chose the iteration method.
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Improved
+
+- [alpha] The `cdf data upload` command can now upload an asset
+hierarchy. It does this by first uploading root assets, then one level
+deep, two level deep, and so on.
+
+## templates
+
+No changes.
+
+

cognite_toolkit-0.6.93/last_version.txt

@@ -0,0 +1 @@
+0.6.92

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.92"
+version = "0.6.93"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/tests/test_unit/test_cdf_tk/test_storageio/test_asset_centric.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+from unittest.mock import MagicMock
 
 import httpx
 import pytest
@@ -23,9 +24,10 @@ from cognite.client.data_classes import (
 from cognite_toolkit._cdf_tk.client import ToolkitClientConfig
 from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client
 from cognite_toolkit._cdf_tk.commands import DownloadCommand, UploadCommand
-from cognite_toolkit._cdf_tk.storageio import AssetIO, EventIO, FileMetadataIO, TimeSeriesIO
+from cognite_toolkit._cdf_tk.storageio import AssetFileReaderAdapter, AssetIO, EventIO, FileMetadataIO, TimeSeriesIO
 from cognite_toolkit._cdf_tk.storageio.selectors import AssetSubtreeSelector, DataSetSelector
 from cognite_toolkit._cdf_tk.utils.collection import chunker
+from cognite_toolkit._cdf_tk.utils.fileio import FileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
@@ -438,3 +440,28 @@ class TestEventIO:
                 uploaded_events.extend(json.loads(call.request.content)["items"])
 
             assert uploaded_events == some_event_data.as_write().dump()
+
+
+class TestAssetFileReaderAdapter:
+    def test_read_assets_chunks(self) -> None:
+        assets = [
+            {"id": 1, "depth": 3},
+            {"id": 2, "depth": 2},
+            {"id": 3, "depth": 1},
+            {"id": 4},
+            {"id": 5, "depth": "not_an_int"},
+        ]
+        assets_with_line_numbers = list(enumerate(assets, start=1))
+        other_reader = MagicMock(spec=FileReader)
+        other_reader.read_chunks_with_line_numbers.return_value = assets_with_line_numbers
+        other_reader.input_file = Path("mocked_file.csv")
+        reader = AssetFileReaderAdapter(other_reader)
+        output = list(reader.read_chunks_with_line_numbers())
+
+        assert output == [
+            (4, {"id": 4}),
+            (5, {"id": 5, "depth": "not_an_int"}),
+            (3, {"id": 3, "depth": 1}),
+            (2, {"id": 2, "depth": 2}),
+            (1, {"id": 1, "depth": 3}),
+        ]

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/tests/test_unit/test_cdf_tk/test_utils/test_fileio.py

@@ -13,6 +13,7 @@ from typing import Any
 import pytest
 
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError, ToolkitValueError
+from cognite_toolkit._cdf_tk.storageio import AssetFileReaderAdapter
 from cognite_toolkit._cdf_tk.utils._auxiliary import get_concrete_subclasses
 from cognite_toolkit._cdf_tk.utils.fileio import (
     COMPRESSION_BY_NAME,
@@ -466,7 +467,11 @@ class TestFileReader:
 
     def test_all_file_readers_registered(self) -> None:
         # YAMLBaseReader is an abstract class, so we exclude it from the expected readers
-        expected_readers = set(get_concrete_subclasses(FileReader)) - {LineReader, YAMLBaseReader}
+        expected_readers = set(get_concrete_subclasses(FileReader)) - {
+            LineReader,
+            YAMLBaseReader,
+            AssetFileReaderAdapter,
+        }
 
         assert set(expected_readers) == set(FILE_READ_CLS_BY_FORMAT.values())
 

{cognite_toolkit-0.6.92 → cognite_toolkit-0.6.93}/uv.lock

@@ -260,7 +260,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.6.92"
+version = "0.6.93"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.6.92/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.6.92"

cognite_toolkit-0.6.92/last_changelog_entry.md

@@ -1,10 +0,0 @@
-## cdf 
-
-### Improved
-
-- [alpha] When downloading assets using `cdf data download`, the
-properties `depth`, `childCount` and `path` are included.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.92/last_git_message.txt

@@ -1,25 +0,0 @@
-[CDF-26219] 👻 Asset include aggregate information (#2179)
-
-# Description
-
-See below:
-<img width="1564" height="200" alt="image"
-src="https://github.com/user-attachments/assets/a2a82d66-bacb-4846-a3ce-736b7459db5b"
-/>
-
-
-## Changelog
-
-- [x] Patch
-- [ ] Skip
-
-## cdf
-
-### Improved
-
-- [alpha] When downloading assets using `cdf data download`, the
-properties `depth`, `childCount` and `path` are included.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.92/last_version.txt

@@ -1 +0,0 @@
-0.6.91