cognite-toolkit 0.6.85__py3-none-any.whl → 0.6.86__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_download.py

@@ -1,4 +1,4 @@
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
 from functools import partial
 from pathlib import Path
 
@@ -42,7 +42,7 @@ class DownloadCommand(ToolkitCommand):
 
         console = io.client.console
         for selector in selectors:
-            target_dir = output_dir / selector.group
+            target_dir = output_dir / sanitize_filename(selector.group)
             if verbose:
                 console.print(f"Downloading {selector.display_name} '{selector!s}' to {target_dir.as_posix()!r}")
 
@@ -57,9 +57,10 @@ class DownloadCommand(ToolkitCommand):
 
             selector.dump_to_file(target_dir)
             columns: list[SchemaColumn] | None = None
-            if file_format in TABLE_WRITE_CLS_BY_FORMAT and isinstance(io, TableStorageIO):
+            is_table = file_format in TABLE_WRITE_CLS_BY_FORMAT
+            if is_table and isinstance(io, TableStorageIO):
                 columns = io.get_schema(selector)
-            elif file_format in TABLE_WRITE_CLS_BY_FORMAT:
+            elif is_table:
                 raise ToolkitValueError(
                     f"Cannot download {selector.kind} in {file_format!r} format. The {selector.kind!r} storage type does not support table schemas."
                 )
@@ -69,7 +70,7 @@ class DownloadCommand(ToolkitCommand):
             ) as writer:
                 executor = ProducerWorkerExecutor[Page[T_CogniteResource], list[dict[str, JsonVal]]](
                     download_iterable=io.stream_data(selector, limit),
-                    process=partial(self.process_data_chunk, io=io, selector=selector),
+                    process=self.create_data_process(io=io, selector=selector, is_table=is_table),
                     write=partial(writer.write_chunks, filestem=filestem),
                     iteration_count=iteration_count,
                     # Limit queue size to avoid filling up memory before the workers can write to disk.
@@ -124,19 +125,20 @@ class DownloadCommand(ToolkitCommand):
         return False
 
     @staticmethod
-    def process_data_chunk(
-        data_page: Page[T_CogniteResource],
+    def create_data_process(
         io: StorageIO[T_Selector, T_CogniteResource],
         selector: T_Selector,
-    ) -> list[dict[str, JsonVal]]:
-        """Processes a chunk of data by converting it to a JSON-compatible format.
+        is_table: bool,
+    ) -> Callable[[Page[T_CogniteResource]], list[dict[str, JsonVal]]]:
+        """Creates a data processing function based on the IO type and whether the output is a table."""
+        if is_table and isinstance(io, TableStorageIO):
 
-        Args:
-            data_page: The page of data to process.
-            io: The StorageIO instance that defines how to process the data.
-            selector: The selection criteria used to identify the data.
+            def row_data_process(chunk: Page[T_CogniteResource]) -> list[dict[str, JsonVal]]:
+                return io.data_to_row(chunk.items, selector)
 
-        Returns:
-            A list of dictionaries representing the processed data in a JSON-compatible format.
-        """
-        return io.data_to_json_chunk(data_page.items, selector)
+            return row_data_process
+
+        def chunk_data_process(data_page: Page[T_CogniteResource]) -> list[dict[str, JsonVal]]:
+            return io.data_to_json_chunk(data_page.items, selector)
+
+        return chunk_data_process

cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -67,7 +67,14 @@ from cognite_toolkit._cdf_tk.utils.useful_types import (
     T_WritableCogniteResourceList,
 )
 
-from ._base import ConfigurableStorageIO, Page, StorageIOConfig, TableStorageIO, UploadableStorageIO, UploadItem
+from ._base import (
+    ConfigurableStorageIO,
+    Page,
+    StorageIOConfig,
+    TableStorageIO,
+    UploadableStorageIO,
+    UploadItem,
+)
 from .selectors import AssetCentricSelector, AssetSubtreeSelector, DataSetSelector
 
 
@@ -179,6 +186,19 @@ class BaseAssetCentricIO(
             asset_ids.update(item.asset_ids or [])
         self.client.lookup.assets.external_id(list(asset_ids))
 
+    def data_to_row(
+        self, data_chunk: Sequence[T_WritableCogniteResource], selector: AssetCentricSelector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        rows: list[dict[str, JsonVal]] = []
+        for chunk in self.data_to_json_chunk(data_chunk, selector):
+            if "metadata" in chunk and isinstance(chunk["metadata"], dict):
+                metadata = chunk.pop("metadata")
+                # MyPy does understand that metadata is a dict here due to the check above.
+                for key, value in metadata.items():  # type: ignore[union-attr]
+                    chunk[f"metadata.{key}"] = value
+            rows.append(chunk)
+        return rows
+
 
 class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetList]):
     KIND = "Assets"

cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -230,3 +230,19 @@ class TableStorageIO(StorageIO[T_Selector, T_CogniteResource], ABC):
 
         """
         raise NotImplementedError()
+
+    @abstractmethod
+    def data_to_row(
+        self, data_chunk: Sequence[T_CogniteResource], selector: T_Selector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        """Convert a chunk of data to a row-based JSON-compatible format.
+
+        Args:
+            data_chunk: The chunk of data to convert, which should be a writable Cognite resource list.
+            selector: Optional selection criteria to identify the data. This is required for some storage types.
+
+        Returns:
+            A list of dictionaries representing the data in a JSON-compatible format.
+
+        """
+        raise NotImplementedError()

cognite_toolkit/_cdf_tk/utils/producer_worker.py

@@ -235,7 +235,7 @@ class ProducerWorkerExecutor(Generic[T_Download, T_Processed]):
                 break
             except Exception as e:
                 self._error_event.set()
-                self.error_message = str(e)
+                self.error_message = f"{type(e).__name__} {e!s}"
                 self.error_traceback = traceback.format_exc()
                 self.console.print(f"[red]Error[/red] occurred while {self.download_description}: {self.error_message}")
                 break
@@ -275,7 +275,7 @@ class ProducerWorkerExecutor(Generic[T_Download, T_Processed]):
                 continue
             except Exception as e:
                 self._error_event.set()
-                self.error_message = str(e)
+                self.error_message = f"{type(e).__name__} {e!s}"
                 self.error_traceback = traceback.format_exc()
                 self.console.print(f"[red]Error[/red] occurred while {self.process_description}: {self.error_message}")
                 break
@@ -297,7 +297,7 @@ class ProducerWorkerExecutor(Generic[T_Download, T_Processed]):
                 continue
             except Exception as e:
                 self._error_event.set()
-                self.error_message = str(e)
+                self.error_message = f"{type(e).__name__} {e!s}"
                 self.error_traceback = traceback.format_exc()
                 self.console.print(f"[red]Error[/red] occurred while {self.write_description}: {self.error_message}")
                 break

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.85
+      image: cognite/toolkit:0.6.86
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.85
+      image: cognite/toolkit:0.6.86
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.85"
+version = "0.6.86"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.85"
+__version__ = "0.6.86"

{cognite_toolkit-0.6.85.dist-info → cognite_toolkit-0.6.86.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.85
+Version: 0.6.86
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.85.dist-info → cognite_toolkit-0.6.86.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=1OSAvbOeuIrnsczEG2BtGqRP3L3sq0VMPthmugnqCUw,5821
-cognite_toolkit/_version.py,sha256=fSvL3Nga6Lczp2I4CiTq0ik_l-HhTaEENmTQBPEDsxU,23
+cognite_toolkit/_version.py,sha256=y8yiu6qqknI_Tf_P0-H0xuN1S1fZ-IRju7amwCqQF3E,23
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=DAUmHf19ByVIGH4MDPdXKHZ0G97CxdD5J-EzHTq66C8,8025
 cognite_toolkit/_cdf_tk/constants.py,sha256=e9XmGvQCqGq7zYQrNoopU5e2KnYZYBPyUC5raGShK7k,6364
@@ -95,7 +95,7 @@ cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=OJYtHiERtUBXm3cjUTyPVaYIMVQp
 cognite_toolkit/_cdf_tk/commands/_base.py,sha256=m2hnXo_AAHhsoSayHZO_zUa4xEt5w5oMB4WCHmJr-AY,2595
 cognite_toolkit/_cdf_tk/commands/_changes.py,sha256=DIwuiRpDhWBDpsW3R3yqj0eWLAE3c_kPbmCaUkxjFuo,24852
 cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMoulWgJzbDC8YIIQDPJ5x3w,1011
-cognite_toolkit/_cdf_tk/commands/_download.py,sha256=qkbzHzd6FZydNiG83vwciBJEmQAG0t9EFfvZb0K89TA,6693
+cognite_toolkit/_cdf_tk/commands/_download.py,sha256=OBKPM_HGGA1i32th1SAgkQM_81CUFvm39kGqBuOeeTs,6816
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
 cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=RadQHsmkPez3fZ5HCP9b82o2_fBx8P_-bTo7prkvWXU,32525
 cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=kXYmP1YMg-JvsuN1iYaMuZH1qZfapya2j-RABGhqGHo,11860
@@ -227,8 +227,8 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Q
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
 cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=aM-skaPnKTH1B7HG0faeTUNf7u1b-sT8l7hh5JRZ1E8,2288
 cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=bhyG1d2_9duPkX-otC2brVcpChvdXSPkYhBHS5T_72g,4343
-cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=spFAdoHxGKDDEWItM4RovSddudwoPIa_uev8WCAQ9zs,27027
-cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=yqAaBlZcE53V_HKcKi_q-OjpC2Nnhkw13cbSvXjx2wY,8682
+cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=d7WfCjuauIyUBfNJTgV5ZN-xSUmLGuuvLB0yPYnEg08,27734
+cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=BQPueVpmxIYtSZ53W4cwp7pFEzvvpBd7Bz_L7Anw8hc,9308
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=_tKOdlo7tMJoh7y-47o7sySfDMRa-G-AFVprmzjn3EQ,9311
 cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=5UkIk5MkyyCyGRYmyD6qTIsEDKbI2EulPesYbqM8qAA,3466
@@ -259,7 +259,7 @@ cognite_toolkit/_cdf_tk/utils/graphql_parser.py,sha256=2i2wDjg_Uw3hJ-pHtPK8hczIu
 cognite_toolkit/_cdf_tk/utils/hashing.py,sha256=3NyNfljyYNTqAyAFBd6XlyWaj43jRzENxIuPdOY6nqo,2116
 cognite_toolkit/_cdf_tk/utils/interactive_select.py,sha256=veV93_O-gATbQ1PfRbZq0VotTgaXA4JcU34j_nLKpSU,36155
 cognite_toolkit/_cdf_tk/utils/modules.py,sha256=9RvOGUaGEi_-A7Qrq0E1tCx82QK8GbvEZXB7r1RnD_U,5974
-cognite_toolkit/_cdf_tk/utils/producer_worker.py,sha256=n01Tr8ml8dHffjX45Vdqmzt2DFGZ0IrcgFembo8y9yM,14153
+cognite_toolkit/_cdf_tk/utils/producer_worker.py,sha256=1l77HIehkq1ARCBH6SlZ_V-jd6QKijYKeWetcUmAXg0,14216
 cognite_toolkit/_cdf_tk/utils/progress_tracker.py,sha256=LGpC22iSTTlo6FWi38kqBu_E4XouTvZU_N953WAzZWA,3865
 cognite_toolkit/_cdf_tk/utils/repository.py,sha256=voQLZ6NiNvdAFxqeWHbvzDLsLHl6spjQBihiLyCsGW8,4104
 cognite_toolkit/_cdf_tk/utils/sentry_utils.py,sha256=Q3ekrR0bWMtlPVQrfUSsETlkLIaDUZ2u-RdNFFr9-dg,564
@@ -284,13 +284,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=oypvpptzzkRZcztIcOxkwipNPTJ54zZGTRUIHwpbTrQ,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=wOg0WNsPFO4Kd14lD9OTlLL7m-W7kF5YbgNZL_uv9zg,2430
-cognite_toolkit/_resources/cdf.toml,sha256=NwEvIORLNsZ8IaxYk-mUhrTuw3b7bKXswk79nxWjRvM,487
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=SKpoFkXR8f0H7AhoDJHmdN-vNkVqmpXzzbhzzNSFGtc,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=BiHNR_3PtLFgSH-Uz7KtKzDkywcsiA_oOjJ4M2it9lg,2430
+cognite_toolkit/_resources/cdf.toml,sha256=vN6PvToANrJEjhUYpEzDdjUa1cSXa9NMgv1mp8iOb2I,487
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.85.dist-info/METADATA,sha256=xmey3u0EyVh4Ef5qgxa0VibUKZ_FLz6jAYQ5opMl_6g,4501
-cognite_toolkit-0.6.85.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.85.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.85.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.85.dist-info/RECORD,,
+cognite_toolkit-0.6.86.dist-info/METADATA,sha256=gVYv6ykmqx89FtC_aWVCshPWVQ3B7LcpBUy1x8y17uQ,4501
+cognite_toolkit-0.6.86.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.86.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.86.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.86.dist-info/RECORD,,