cognite-toolkit 0.6.85__tar.gz → 0.6.86__tar.gz

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.85
+Version: 0.6.86
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cdf.toml

@@ -31,7 +31,7 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.85"
+version = "0.6.86"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cognite_toolkit/_cdf_tk/commands/_download.py

@@ -1,4 +1,4 @@
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
 from functools import partial
 from pathlib import Path
 
@@ -42,7 +42,7 @@ class DownloadCommand(ToolkitCommand):
 
         console = io.client.console
         for selector in selectors:
-            target_dir = output_dir / selector.group
+            target_dir = output_dir / sanitize_filename(selector.group)
             if verbose:
                 console.print(f"Downloading {selector.display_name} '{selector!s}' to {target_dir.as_posix()!r}")
 
@@ -57,9 +57,10 @@ class DownloadCommand(ToolkitCommand):
 
             selector.dump_to_file(target_dir)
             columns: list[SchemaColumn] | None = None
-            if file_format in TABLE_WRITE_CLS_BY_FORMAT and isinstance(io, TableStorageIO):
+            is_table = file_format in TABLE_WRITE_CLS_BY_FORMAT
+            if is_table and isinstance(io, TableStorageIO):
                 columns = io.get_schema(selector)
-            elif file_format in TABLE_WRITE_CLS_BY_FORMAT:
+            elif is_table:
                 raise ToolkitValueError(
                     f"Cannot download {selector.kind} in {file_format!r} format. The {selector.kind!r} storage type does not support table schemas."
                 )
@@ -69,7 +70,7 @@ class DownloadCommand(ToolkitCommand):
             ) as writer:
                 executor = ProducerWorkerExecutor[Page[T_CogniteResource], list[dict[str, JsonVal]]](
                     download_iterable=io.stream_data(selector, limit),
-                    process=partial(self.process_data_chunk, io=io, selector=selector),
+                    process=self.create_data_process(io=io, selector=selector, is_table=is_table),
                     write=partial(writer.write_chunks, filestem=filestem),
                     iteration_count=iteration_count,
                     # Limit queue size to avoid filling up memory before the workers can write to disk.
@@ -124,19 +125,20 @@ class DownloadCommand(ToolkitCommand):
         return False
 
     @staticmethod
-    def process_data_chunk(
-        data_page: Page[T_CogniteResource],
+    def create_data_process(
         io: StorageIO[T_Selector, T_CogniteResource],
         selector: T_Selector,
-    ) -> list[dict[str, JsonVal]]:
-        """Processes a chunk of data by converting it to a JSON-compatible format.
+        is_table: bool,
+    ) -> Callable[[Page[T_CogniteResource]], list[dict[str, JsonVal]]]:
+        """Creates a data processing function based on the IO type and whether the output is a table."""
+        if is_table and isinstance(io, TableStorageIO):
 
-        Args:
-            data_page: The page of data to process.
-            io: The StorageIO instance that defines how to process the data.
-            selector: The selection criteria used to identify the data.
+            def row_data_process(chunk: Page[T_CogniteResource]) -> list[dict[str, JsonVal]]:
+                return io.data_to_row(chunk.items, selector)
 
-        Returns:
-            A list of dictionaries representing the processed data in a JSON-compatible format.
-        """
-        return io.data_to_json_chunk(data_page.items, selector)
+            return row_data_process
+
+        def chunk_data_process(data_page: Page[T_CogniteResource]) -> list[dict[str, JsonVal]]:
+            return io.data_to_json_chunk(data_page.items, selector)
+
+        return chunk_data_process

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -67,7 +67,14 @@ from cognite_toolkit._cdf_tk.utils.useful_types import (
     T_WritableCogniteResourceList,
 )
 
-from ._base import ConfigurableStorageIO, Page, StorageIOConfig, TableStorageIO, UploadableStorageIO, UploadItem
+from ._base import (
+    ConfigurableStorageIO,
+    Page,
+    StorageIOConfig,
+    TableStorageIO,
+    UploadableStorageIO,
+    UploadItem,
+)
 from .selectors import AssetCentricSelector, AssetSubtreeSelector, DataSetSelector
 
 
@@ -179,6 +186,19 @@ class BaseAssetCentricIO(
             asset_ids.update(item.asset_ids or [])
         self.client.lookup.assets.external_id(list(asset_ids))
 
+    def data_to_row(
+        self, data_chunk: Sequence[T_WritableCogniteResource], selector: AssetCentricSelector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        rows: list[dict[str, JsonVal]] = []
+        for chunk in self.data_to_json_chunk(data_chunk, selector):
+            if "metadata" in chunk and isinstance(chunk["metadata"], dict):
+                metadata = chunk.pop("metadata")
+                # MyPy does understand that metadata is a dict here due to the check above.
+                for key, value in metadata.items():  # type: ignore[union-attr]
+                    chunk[f"metadata.{key}"] = value
+            rows.append(chunk)
+        return rows
+
 
 class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetList]):
     KIND = "Assets"

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -230,3 +230,19 @@ class TableStorageIO(StorageIO[T_Selector, T_CogniteResource], ABC):
 
         """
         raise NotImplementedError()
+
+    @abstractmethod
+    def data_to_row(
+        self, data_chunk: Sequence[T_CogniteResource], selector: T_Selector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        """Convert a chunk of data to a row-based JSON-compatible format.
+
+        Args:
+            data_chunk: The chunk of data to convert, which should be a writable Cognite resource list.
+            selector: Optional selection criteria to identify the data. This is required for some storage types.
+
+        Returns:
+            A list of dictionaries representing the data in a JSON-compatible format.
+
+        """
+        raise NotImplementedError()

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cognite_toolkit/_cdf_tk/utils/producer_worker.py

@@ -235,7 +235,7 @@ class ProducerWorkerExecutor(Generic[T_Download, T_Processed]):
                 break
             except Exception as e:
                 self._error_event.set()
-                self.error_message = str(e)
+                self.error_message = f"{type(e).__name__} {e!s}"
                 self.error_traceback = traceback.format_exc()
                 self.console.print(f"[red]Error[/red] occurred while {self.download_description}: {self.error_message}")
                 break
@@ -275,7 +275,7 @@ class ProducerWorkerExecutor(Generic[T_Download, T_Processed]):
                 continue
             except Exception as e:
                 self._error_event.set()
-                self.error_message = str(e)
+                self.error_message = f"{type(e).__name__} {e!s}"
                 self.error_traceback = traceback.format_exc()
                 self.console.print(f"[red]Error[/red] occurred while {self.process_description}: {self.error_message}")
                 break
@@ -297,7 +297,7 @@ class ProducerWorkerExecutor(Generic[T_Download, T_Processed]):
                 continue
             except Exception as e:
                 self._error_event.set()
-                self.error_message = str(e)
+                self.error_message = f"{type(e).__name__} {e!s}"
                 self.error_traceback = traceback.format_exc()
                 self.console.print(f"[red]Error[/red] occurred while {self.write_description}: {self.error_message}")
                 break

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.85
+      image: cognite/toolkit:0.6.86
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.85
+      image: cognite/toolkit:0.6.86
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.85"
+version = "0.6.86"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.86/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.86"

cognite_toolkit-0.6.86/last_changelog_entry.md

@@ -0,0 +1,13 @@
+## cdf 
+
+### Fixed
+
+- [alpha] When running `cdf data download`, Toolkit now sanitize the
+name for the data directory it creates.
+- [alpha] When running `cdf data download assets` no longer skips
+metdata when a `.csv` or `.parquet` is selected as the
+  output format. 
+
+## templates
+
+No changes.

cognite_toolkit-0.6.86/last_git_message.txt

@@ -0,0 +1,24 @@
+[CDF-26070] 🔨 Download table format (#2152)
+
+# Description
+
+Testing the upload /download functionality and finding issues. 
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Fixed
+
+- [alpha] When running `cdf data download`, Toolkit now sanitize the
+name for the data directory it creates.
+- [alpha] When running `cdf data download assets` no longer skips
+metdata when a `.csv` or `.parquet` is selected as the
+  output format. 
+
+## templates
+
+No changes.

cognite_toolkit-0.6.86/last_version.txt

@@ -0,0 +1 @@
+0.6.85

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.85"
+version = "0.6.86"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

cognite_toolkit-0.6.86/tests/test_unit/test_cdf_tk/test_commands/test_download.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+
+from cognite.client.data_classes import Asset, AssetList, TransformationPreviewResult
+
+from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client
+from cognite_toolkit._cdf_tk.commands import DownloadCommand
+from cognite_toolkit._cdf_tk.storageio import AssetIO
+from cognite_toolkit._cdf_tk.storageio.selectors import DataSetSelector
+from cognite_toolkit._cdf_tk.utils.fileio import CSVReader
+
+
+class TestDownloadCommand:
+    def test_download_data_in_table_format(self, tmp_path: Path) -> None:
+        cmd = DownloadCommand(silent=True, skip_tracking=True)
+        with monkeypatch_toolkit_client() as client:
+            dataset = "my/:_data_set"
+            client.assets.aggregate_count.return_value = 1
+            client.assets.return_value = [
+                AssetList([Asset(id=123, name="asset_123", metadata={"key": "value"}, data_set_id=42)])
+            ]
+            client.lookup.data_sets.id.return_value = 42
+            client.lookup.data_sets.external_id.return_value = dataset
+            client.transformations.preview.return_value = TransformationPreviewResult(
+                None, results=[{"key": "key", "key_count": 1}]
+            )
+
+            cmd.download(
+                selectors=[DataSetSelector(kind="Assets", data_set_external_id=dataset)],
+                io=AssetIO(client=client),
+                output_dir=tmp_path,
+                verbose=True,
+                file_format=".csv",
+                compression="none",
+                limit=-1,
+            )
+
+            csv_files = list(tmp_path.rglob("*.csv"))
+            assert len(csv_files) == 1
+            content = list(CSVReader(csv_files[0]).read_chunks_unprocessed())
+            assert content == [
+                {
+                    "dataSetExternalId": dataset,
+                    "description": "",
+                    "externalId": "",
+                    "geoLocation": "",
+                    "labels": "",
+                    "metadata.key": "value",
+                    "name": "asset_123",
+                    "parentExternalId": "",
+                    "source": "",
+                }
+            ]

{cognite_toolkit-0.6.85 → cognite_toolkit-0.6.86}/uv.lock

@@ -260,7 +260,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.6.85"
+version = "0.6.86"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.6.85/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.6.85"

cognite_toolkit-0.6.85/last_changelog_entry.md

@@ -1,13 +0,0 @@
-## cdf 
-
-### Improved
-
-- The `cdf purge dataset` has been reworked. It now gives progress bar
-when deleting. It is also robust against individual items failing to
-delete. [alpha] If you enable the alpha flag `v07=true` you will get
-further options like deleting assets recursively, select whether to
-delete only data or only configurations, or both.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.85/last_git_message.txt

@@ -1,37 +0,0 @@
-[CDF-25834] 🏰 Rework purge dataset Part 4 (#2148)
-
-# Description
-
-This is part 1 of reimplementing the cdf purge dataset command. 
-
-This PR set up the CLI interface and remove the old implementation.
-
-The motivation for this reimplementation is as follows:
-* We have new components such as HTTPClient and WorkerProducer that will
-be very useful for running the purge code.
-* The current implementation lacks proper testing, a reimplementation
-will ensure it is well tested.
-
-<img width="1872" height="383" alt="image"
-src="https://github.com/user-attachments/assets/0a1d4e12-7996-48bb-aca9-ef5c4f9898ce"
-/>
-
-
-## Changelog
-
-- [x] Patch
-- [ ] Skip
-
-## cdf
-
-### Improved
-
-- The `cdf purge dataset` has been reworked. It now gives progress bar
-when deleting. It is also robust against individual items failing to
-delete. [alpha] If you enable the alpha flag `v07=true` you will get
-further options like deleting assets recursively, select whether to
-delete only data or only configurations, or both.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.85/last_version.txt

@@ -1 +0,0 @@
-0.6.84