cognite-toolkit 0.7.8__py3-none-any.whl → 0.7.9__py3-none-any.whl

cognite_toolkit/_cdf_tk/storageio/_raw.py

@@ -1,4 +1,5 @@
 from collections.abc import Iterable, Sequence
+from itertools import chain
 from uuid import uuid4
 
 from cognite.client.data_classes import Row, RowWrite
@@ -6,6 +7,8 @@ from cognite.client.data_classes import Row, RowWrite
 from cognite_toolkit._cdf_tk.cruds import RawDatabaseCRUD, RawTableCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.utils import sanitize_filename
+from cognite_toolkit._cdf_tk.utils.collection import chunker
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemsRequest
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
@@ -96,3 +99,26 @@ class RawIO(
         if selector is not None and selector.key is not None and selector.key in row:
             key = str(row.pop(selector.key))
         return RowWrite(key=key, columns=row)
+
+    @classmethod
+    def read_chunks(
+        cls, reader: MultiFileReader, selector: RawTableSelector
+    ) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
+        if not reader.is_table or selector.key is None:
+            yield from super().read_chunks(reader, selector)
+            return
+        data_name = "row" if reader.is_table else "line"
+        # Validate that the key exists in all files
+        for input_file in sorted(reader.input_files, key=reader._part_no):
+            iterable = reader.reader_class(input_file).read_chunks()
+            try:
+                first = next(iterable)
+            except StopIteration:
+                continue
+            if selector.key not in first:
+                raise ToolkitValueError(
+                    f"Column '{selector.key}' not found in file {input_file.as_posix()!r}. Please ensure the specified column exists."
+                )
+            full_iterator = chain([first], iterable)
+            line_numbered_iterator = ((f"{data_name} {i}", row) for i, row in enumerate(full_iterator, start=1))
+            yield from chunker(line_numbered_iterator, cls.CHUNK_SIZE)

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.8
+      image: cognite/toolkit:0.7.9
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.8
+      image: cognite/toolkit:0.7.9
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.8"
+version = "0.7.9"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.8"
+__version__ = "0.7.9"

{cognite_toolkit-0.7.8.dist-info → cognite_toolkit-0.7.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.8
+Version: 0.7.9
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.7.8.dist-info → cognite_toolkit-0.7.9.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=PzDig6dgbDX5VL88AeijQuTeYb2SS_yvenw9gr4fnxY,5794
-cognite_toolkit/_version.py,sha256=uC8wB9mRblQ0jUBAOUyCQLUQJ39MC2xybVLB_8ZsevU,22
+cognite_toolkit/_version.py,sha256=Plj3sh67uLqQt2X1uejJ0B3ggLdj4Fr1HPHcgyqtneU,22
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
 cognite_toolkit/_cdf_tk/constants.py,sha256=TplKm2J9pGRHq7nAnLI0caTMHetS04OIz3hfq-jvGzo,7236
@@ -249,7 +249,7 @@ cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVME
 cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=AGTQm9CBRbu1oXbBh0X7UGzFrHnlWZExqNvAohT0hM0,8641
 cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=g8HDDHbBPBcb3g6Zp-vHc8uvNZYrZXWs5G0jq_eOc4w,10884
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
-cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=5WjAFiVR0KKRhMqCy1IRy1TQFWj86D7nGu5WSFNLp6U,3869
+cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=pgZN5MbqDwMZl9Ow1KouDJUO2Ngga8_b6hwv7H31SVQ,5161
 cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=ELUCirQmhDR52PVIhLRd_2M1DWYURwL44U6WiN9_hEA,2225
 cognite_toolkit/_cdf_tk/storageio/selectors/_asset_centric.py,sha256=7Iv_ccVX6Vzt3ZLFZ0Er3hN92iEsFTm9wgF-yermOWE,1467
 cognite_toolkit/_cdf_tk/storageio/selectors/_base.py,sha256=hjFkbmNGsK3QIW-jnJV_8YNmvVROERxzG82qIZhU7SM,3065
@@ -303,13 +303,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=L55NhSDSUlxT5pZbTRphvOlO0MvCiuWXkNsGq4q7Bn4,666
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=9i3QNPhYewexcukRhRUaXIFh8Xgd4k8XxTiFWKxyD64,2429
-cognite_toolkit/_resources/cdf.toml,sha256=TN3TRIdZo6zfKthRLj0JW7VatUQ_pLeKUktmm2zl_1E,474
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=1_fEjrhRKAHXIqLW4lBT6JS9GmVbyIxZFK1tYku7RA0,666
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=3yRJMs7Um0M9HikqM8RlGbnVK7jqdJyEgZrbkfIqMHc,2429
+cognite_toolkit/_resources/cdf.toml,sha256=3vbfRsbDoAwloTprZxOV1lEmobT1VXoNAHniNXimJkI,474
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.7.8.dist-info/METADATA,sha256=LCh--Lkkr8NwKHeApUv_HfSm_9p2e1wuMQnMmP76zYQ,4500
-cognite_toolkit-0.7.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cognite_toolkit-0.7.8.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.7.8.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.7.8.dist-info/RECORD,,
+cognite_toolkit-0.7.9.dist-info/METADATA,sha256=dCFxCZZyD0iS8h4oBEWmWX7DuM7e9r8iIop0eSha7eY,4500
+cognite_toolkit-0.7.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cognite_toolkit-0.7.9.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.7.9.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.7.9.dist-info/RECORD,,