cognite-toolkit 0.6.91__tar.gz → 0.6.93__tar.gz

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
           - --fixable=E,W,F,I,T,RUF,TID,UP
           - --target-version=py310
       - id: ruff-format
-    rev: v0.14.3
+    rev: v0.14.4
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.45.0

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.91
+Version: 0.6.93
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cdf.toml

@@ -31,7 +31,7 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.91"
+version = "0.6.93"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -9,7 +9,14 @@ from rich.console import Console
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
-from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, are_same_kind, get_upload_io
+from cognite_toolkit._cdf_tk.storageio import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    T_Selector,
+    UploadableStorageIO,
+    are_same_kind,
+    get_upload_io,
+)
 from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
 from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
 from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
@@ -182,11 +189,18 @@ class UploadCommand(ToolkitCommand):
                     is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
                     if is_table and not isinstance(io, TableUploadableStorageIO):
                         raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+                    if isinstance(io, AssetIO):
+                        # Assets needs to be uploaded from the root asset to the leaves. The specialized reader
+                        # handles this by iterating over the file n times, where n is the depth of the asset hierarchy.
+                        reader = AssetFileReaderAdapter(reader)
                     tracker = ProgressTracker[str]([self._UPLOAD])
                     data_name = "row" if is_table else "line"
                     executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
                         download_iterable=chunker(
-                            ((f"{data_name} {line_no}", item) for line_no, item in enumerate(reader.read_chunks(), 1)),
+                            (
+                                (f"{data_name} {line_no}", item)
+                                for line_no, item in reader.read_chunks_with_line_numbers()
+                            ),
                             io.CHUNK_SIZE,
                         ),
                         process=partial(io.rows_to_data, selector=selector)

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py

@@ -6,6 +6,7 @@ from typing import Any, final
 
 import pandas as pd
 from cognite.client.data_classes import (
+    AggregateResultItem,
     Asset,
     AssetList,
     AssetWrite,
@@ -213,6 +214,20 @@ class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset, AssetWriteList, AssetList])
             dumped.pop("metadata", None)
         if "parentId" in dumped and "parentId" not in local:
             dumped.pop("parentId")
+        if resource.aggregates:
+            # This is only included when the asset is downloaded/migrated with aggregated properties
+            aggregates = (
+                resource.aggregates.dump()
+                if isinstance(resource.aggregates, AggregateResultItem)
+                else resource.aggregates
+            )
+            if "path" in aggregates:
+                path = aggregates.pop("path", [])
+                if path:
+                    aggregates["path"] = self.client.lookup.assets.external_id(
+                        [segment["id"] for segment in path if "id" in segment]
+                    )
+            dumped.update(aggregates)
         return dumped
 
 

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/storageio/__init__.py

@@ -5,7 +5,15 @@ from cognite_toolkit._cdf_tk.utils.fileio import COMPRESSION_BY_SUFFIX
 
 from ._annotations import FileAnnotationIO
 from ._applications import CanvasIO, ChartIO
-from ._asset_centric import AssetIO, BaseAssetCentricIO, EventIO, FileMetadataIO, HierarchyIO, TimeSeriesIO
+from ._asset_centric import (
+    AssetFileReaderAdapter,
+    AssetIO,
+    BaseAssetCentricIO,
+    EventIO,
+    FileMetadataIO,
+    HierarchyIO,
+    TimeSeriesIO,
+)
 from ._base import (
     ConfigurableStorageIO,
     Page,
@@ -45,6 +53,7 @@ def are_same_kind(kind: str, kind_or_path: str | Path, /) -> bool:
 
 
 __all__ = [
+    "AssetFileReaderAdapter",
     "AssetIO",
     "BaseAssetCentricIO",
     "CanvasIO",

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -1,9 +1,11 @@
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from collections.abc import Iterable, MutableSequence, Sequence
+from collections.abc import Iterable, Iterator, MutableSequence, Sequence
+from io import TextIOWrapper
 from typing import ClassVar, Generic
 
 from cognite.client.data_classes import (
+    AggregateResultItem,
     Asset,
     AssetList,
     AssetWrite,
@@ -47,7 +49,7 @@ from cognite_toolkit._cdf_tk.utils.aggregators import (
     TimeSeriesAggregator,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import metadata_key_counts
-from cognite_toolkit._cdf_tk.utils.fileio import SchemaColumn
+from cognite_toolkit._cdf_tk.utils.fileio import FileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.http_client import (
     FailedRequestItems,
     FailedRequestMessage,
@@ -267,6 +269,9 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
             SchemaColumn(name="source", type="string"),
             SchemaColumn(name="labels", type="string", is_array=True),
             SchemaColumn(name="geoLocation", type="json"),
+            SchemaColumn(name="childCount", type="integer"),
+            SchemaColumn(name="depth", type="integer"),
+            SchemaColumn(name="path", type="string", is_array=True),
         ]
         return asset_schema + metadata_schema
 
@@ -277,6 +282,7 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
             limit=limit,
             asset_subtree_external_ids=asset_subtree_external_ids,
             data_set_external_ids=data_set_external_ids,
+            aggregated_properties=["child_count", "path", "depth"],
         ):
             self._collect_dependencies(asset_list, selector)
             yield Page(worker_id="main", items=asset_list)
@@ -287,6 +293,14 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
         # Ensure data sets are looked up to populate cache.
         # This is to avoid looking up each data set id individually in the .dump_resource call.
         self._populate_data_set_cache(data_chunk)
+        asset_ids = {
+            segment["id"]
+            for item in data_chunk
+            if isinstance(item.aggregates, AggregateResultItem)
+            for segment in item.aggregates.path or []
+            if "id" in segment
+        }
+        self.client.lookup.assets.external_id(list(asset_ids))
 
         return [self._crud.dump_resource(item) for item in data_chunk]
 
@@ -626,3 +640,44 @@ class HierarchyIO(ConfigurableStorageIO[AssetCentricSelector, AssetCentricResour
 
     def get_resource_io(self, kind: str) -> BaseAssetCentricIO:
         return self._io_by_kind[kind]
+
+
+class AssetFileReaderAdapter(FileReader):
+    """Adapter of the FileReader to read asset-centric data.
+
+    This is used when uploading assets from files to account for the hierarchical structure. It returns the assets
+    by the depth in the hierarchy, starting from the root assets and going down to the leaf assets.
+
+    Args:
+        other_reader (FileReader): The underlying FileReader to read data from.
+    """
+
+    def __init__(self, other_reader: FileReader) -> None:
+        super().__init__(other_reader.input_file)
+        self._other_reader = other_reader
+
+    def read_chunks(self) -> Iterator[dict[str, JsonVal]]:
+        """Reads chunks from the file, yielding each chunk, sorted by asset depth."""
+        yield from (item for _, item in self.read_chunks_with_line_numbers())
+
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        current_depth = max_depth = 0
+        while current_depth <= max_depth:
+            for line_number, item in self._other_reader.read_chunks_with_line_numbers():
+                try:
+                    depth = int(item.get("depth"))  # type: ignore[arg-type]
+                except (TypeError, ValueError):
+                    if current_depth == 0:
+                        # If depth is not set, we yield it at depth 0
+                        yield line_number, item
+                    continue
+                if depth == current_depth:
+                    yield line_number, item
+                elif current_depth == 0:
+                    max_depth = max(max_depth, depth)
+            current_depth += 1
+
+    def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
+        # This method is not used by AssetFileReaderAdapter as read_chunks is overridden.
+        # It is implemented to satisfy the abstract base class, but should not be called.
+        raise NotImplementedError(f"{type(self).__name__} does not implement '_read_chunks_from_file'.")

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_cdf_tk/utils/fileio/_readers.py

@@ -30,6 +30,10 @@ class FileReader(FileIO, ABC):
         with compression.open("r") as file:
             yield from self._read_chunks_from_file(file)
 
+    def read_chunks_with_line_numbers(self) -> Iterator[tuple[int, dict[str, JsonVal]]]:
+        """Read chunks from the file, yielding each chunk with its corresponding line number."""
+        yield from enumerate(self.read_chunks(), start=1)
+
     @abstractmethod
     def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
         """Read chunks from the file."""

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.91
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.91
+      image: cognite/toolkit:0.6.93
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.91"
+version = "0.6.93"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.93/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.93"

cognite_toolkit-0.6.93/last_changelog_entry.md

@@ -0,0 +1,11 @@
+## cdf 
+
+### Improved
+
+- [alpha] The `cdf data upload` command can now upload an asset
+hierarchy. It does this by first uploading root assets, then one level
+deep, two level deep, and so on.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.93/last_git_message.txt

@@ -0,0 +1,27 @@
+[CDF-26220, CDF-26218] 🤖 Upload assets hierarchy (#2180)
+
+# Description
+
+Handling uploading of asset hierarchy by iterating over the file
+multiple times. Note that the alternative would be to do a sort in
+memory, but the data plugin is optimized for not exploding in memory
+thus chose the iteration method.
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Improved
+
+- [alpha] The `cdf data upload` command can now upload an asset
+hierarchy. It does this by first uploading root assets, then one level
+deep, two level deep, and so on.
+
+## templates
+
+No changes.
+
+

cognite_toolkit-0.6.93/last_version.txt

@@ -0,0 +1 @@
+0.6.92

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.91"
+version = "0.6.93"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/tests/test_integration/test_commands/test_download_command.py

@@ -3,10 +3,11 @@ from pathlib import Path
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands import DownloadCommand
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM
-from cognite_toolkit._cdf_tk.storageio import RawIO
-from cognite_toolkit._cdf_tk.storageio.selectors import RawTableSelector, SelectedTable
+from cognite_toolkit._cdf_tk.storageio import AssetIO, RawIO
+from cognite_toolkit._cdf_tk.storageio.selectors import DataSetSelector, RawTableSelector, SelectedTable
 from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
-from cognite_toolkit._cdf_tk.utils.fileio import NDJsonReader
+from cognite_toolkit._cdf_tk.utils.fileio import CSVReader, NDJsonReader
+from tests.test_integration.conftest import HierarchyMinimal
 from tests.test_integration.constants import TIMESERIES_COUNT, TIMESERIES_TABLE
 
 
@@ -39,3 +40,27 @@ class TestDownloadCommand:
         assert database_file is not None, f"Database configuration file not found. Found config files: {config_files}"
         dumped_db = read_yaml_file(database_file, "dict")
         assert dumped_db == {"dbName": table.table.db_name}
+
+    def test_download_assets(
+        self, toolkit_client: ToolkitClient, migration_hierarchy_minimal: HierarchyMinimal, tmp_path: Path
+    ) -> None:
+        cmd = DownloadCommand(silent=True, skip_tracking=True)
+        selected = DataSetSelector(kind="Assets", data_set_external_id=migration_hierarchy_minimal.dataset.external_id)
+        cmd.download(
+            [selected],
+            AssetIO(toolkit_client),
+            output_dir=tmp_path,
+            verbose=False,
+            file_format=".csv",
+            compression="none",
+            limit=None,
+        )
+        downloaded_files = list(tmp_path.rglob("*.csv"))
+        assert len(downloaded_files) == 1, "Expected exactly one file to be downloaded."
+        chunks = list(CSVReader(downloaded_files[0]).read_chunks_unprocessed())
+        assert len(chunks) == 2, f"Expected 2 assets, got {len(chunks)}."
+        columns = set(chunks[0].keys())
+        expected_aggregate_columns = {"depth", "childCount", "path"}
+        assert expected_aggregate_columns <= columns, (
+            f"Expected hierarchy columns missing in downloaded data: {columns}"
+        )

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/tests/test_unit/test_cdf_tk/test_commands/test_download.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from cognite.client.data_classes import Asset, AssetList, TransformationPreviewResult
+from cognite.client.data_classes import AggregateResultItem, Asset, AssetList, TransformationPreviewResult
 
 from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client
 from cognite_toolkit._cdf_tk.commands import DownloadCommand
@@ -16,7 +16,17 @@ class TestDownloadCommand:
             dataset = "my/:_data_set"
             client.assets.aggregate_count.return_value = 1
             client.assets.return_value = [
-                AssetList([Asset(id=123, name="asset_123", metadata={"key": "value"}, data_set_id=42)])
+                AssetList(
+                    [
+                        Asset(
+                            id=123,
+                            name="asset_123",
+                            metadata={"key": "value"},
+                            data_set_id=42,
+                            aggregates=AggregateResultItem(1, depth=0),
+                        )
+                    ]
+                )
             ]
             client.lookup.data_sets.id.return_value = 42
             client.lookup.data_sets.external_id.return_value = dataset
@@ -48,5 +58,8 @@ class TestDownloadCommand:
                     "name": "asset_123",
                     "parentExternalId": "",
                     "source": "",
+                    "childCount": "1",
+                    "depth": "0",
+                    "path": "",
                 }
             ]

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/tests/test_unit/test_cdf_tk/test_storageio/test_asset_centric.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+from unittest.mock import MagicMock
 
 import httpx
 import pytest
@@ -23,9 +24,10 @@ from cognite.client.data_classes import (
 from cognite_toolkit._cdf_tk.client import ToolkitClientConfig
 from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client
 from cognite_toolkit._cdf_tk.commands import DownloadCommand, UploadCommand
-from cognite_toolkit._cdf_tk.storageio import AssetIO, EventIO, FileMetadataIO, TimeSeriesIO
+from cognite_toolkit._cdf_tk.storageio import AssetFileReaderAdapter, AssetIO, EventIO, FileMetadataIO, TimeSeriesIO
 from cognite_toolkit._cdf_tk.storageio.selectors import AssetSubtreeSelector, DataSetSelector
 from cognite_toolkit._cdf_tk.utils.collection import chunker
+from cognite_toolkit._cdf_tk.utils.fileio import FileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
@@ -438,3 +440,28 @@ class TestEventIO:
                 uploaded_events.extend(json.loads(call.request.content)["items"])
 
             assert uploaded_events == some_event_data.as_write().dump()
+
+
+class TestAssetFileReaderAdapter:
+    def test_read_assets_chunks(self) -> None:
+        assets = [
+            {"id": 1, "depth": 3},
+            {"id": 2, "depth": 2},
+            {"id": 3, "depth": 1},
+            {"id": 4},
+            {"id": 5, "depth": "not_an_int"},
+        ]
+        assets_with_line_numbers = list(enumerate(assets, start=1))
+        other_reader = MagicMock(spec=FileReader)
+        other_reader.read_chunks_with_line_numbers.return_value = assets_with_line_numbers
+        other_reader.input_file = Path("mocked_file.csv")
+        reader = AssetFileReaderAdapter(other_reader)
+        output = list(reader.read_chunks_with_line_numbers())
+
+        assert output == [
+            (4, {"id": 4}),
+            (5, {"id": 5, "depth": "not_an_int"}),
+            (3, {"id": 3, "depth": 1}),
+            (2, {"id": 2, "depth": 2}),
+            (1, {"id": 1, "depth": 3}),
+        ]

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/tests/test_unit/test_cdf_tk/test_utils/test_fileio.py

@@ -13,6 +13,7 @@ from typing import Any
 import pytest
 
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError, ToolkitValueError
+from cognite_toolkit._cdf_tk.storageio import AssetFileReaderAdapter
 from cognite_toolkit._cdf_tk.utils._auxiliary import get_concrete_subclasses
 from cognite_toolkit._cdf_tk.utils.fileio import (
     COMPRESSION_BY_NAME,
@@ -466,7 +467,11 @@ class TestFileReader:
 
     def test_all_file_readers_registered(self) -> None:
         # YAMLBaseReader is an abstract class, so we exclude it from the expected readers
-        expected_readers = set(get_concrete_subclasses(FileReader)) - {LineReader, YAMLBaseReader}
+        expected_readers = set(get_concrete_subclasses(FileReader)) - {
+            LineReader,
+            YAMLBaseReader,
+            AssetFileReaderAdapter,
+        }
 
         assert set(expected_readers) == set(FILE_READ_CLS_BY_FORMAT.values())
 

{cognite_toolkit-0.6.91 → cognite_toolkit-0.6.93}/uv.lock

@@ -260,7 +260,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.6.91"
+version = "0.6.93"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.6.91/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.6.91"

cognite_toolkit-0.6.91/last_changelog_entry.md

@@ -1,10 +0,0 @@
-## cdf 
-
-### Added
-
-- [alpha] Support for Infield Location Config in the `cdf build/deploy`
-methods.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.91/last_git_message.txt

@@ -1,33 +0,0 @@
-[CDF-26172, CDF-26170]  😃Infield CDM V1 CRUD (#2181)
-
-# Description
-
-Adds support for the Infield CDM V1 CRUD.
-
-We are moving away from depending on internals of the `cognite-sdk`, and
-this is the first resource used in a CRUD implementation that is not
-using the `CogniteResource` as defined by the `cognite-sdk`. To support
-the new resource classes, I introduced [these
-protocols](https://github.com/cognitedata/toolkit/pull/2181/files#diff-beedbbdc48da38bb10f4d3a05c2db6dd2b680d677ee725dfc7b0268af6b287bbR14)
-. This is ducktyping of the resources used by the CRUD classes such that
-we not have to subclass `CogniteResource`, `CogniteResourceList`, and so
-on. Note if we decide to remove the `cognit-sdk` as a dependency these
-protocols can be replaced by the pydantic classe ResoruceRequest and
-ResourceResponse.
-
-
-## Changelog
-
-- [x] Patch
-- [ ] Skip
-
-## cdf
-
-### Added
-
-- [alpha] Support for Infield Location Config in the `cdf build/deploy`
-methods.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.91/last_version.txt

@@ -1 +0,0 @@
-0.6.90