cognite-toolkit 0.7.20__py3-none-any.whl → 0.7.21__py3-none-any.whl

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py

@@ -437,4 +437,5 @@ class InFieldCDMLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InFieldCDMLocati
         elif len(json_path) == 4 and json_path[:2] == ("dataExplorationConfig", "filters") and json_path[3] == "values":
             # Handles dataExplorationConfig.filters[i].values
             return diff_list_hashable(local, cdf)
+
         return super().diff_list(local, cdf, json_path)

cognite_toolkit/_cdf_tk/storageio/_file_content.py

@@ -12,7 +12,9 @@ from cognite.client.data_classes.data_modeling import NodeId, ViewId
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.cruds import FileMetadataCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
-from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
+from cognite_toolkit._cdf_tk.protocols import ResourceResponseProtocol
+from cognite_toolkit._cdf_tk.utils import sanitize_filename
+from cognite_toolkit._cdf_tk.utils.collection import chunker, chunker_sequence
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import (
     DataBodyRequest,
@@ -35,6 +37,7 @@ from .selectors._file_content import (
     FileIdentifier,
     FileInstanceID,
     FileInternalID,
+    FileTemplateSelector,
 )
 from .selectors._file_content import NodeId as SelectorNodeId
 
@@ -47,7 +50,16 @@ class UploadFileContentItem(UploadItem[FileMetadataWrite]):
     mime_type: str
 
 
-class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileMetadataWrite]):
+@dataclass
+class MetadataWithFilePath(ResourceResponseProtocol):
+    metadata: FileMetadata
+    file_path: Path
+
+    def as_write(self) -> FileMetadataWrite:
+        return self.metadata.as_write()
+
+
+class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePath, FileMetadataWrite]):
     SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
     SUPPORTED_COMPRESSIONS = frozenset({".gz"})
     CHUNK_SIZE = 10
@@ -61,10 +73,12 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
         self._crud = FileMetadataCRUD(client, None, None)
         self._target_dir = target_dir
 
-    def as_id(self, item: FileMetadata) -> str:
-        return item.external_id or str(item.id)
+    def as_id(self, item: MetadataWithFilePath) -> str:
+        return item.metadata.external_id or str(item.metadata.id)
 
-    def stream_data(self, selector: FileContentSelector, limit: int | None = None) -> Iterable[Page[FileMetadata]]:
+    def stream_data(
+        self, selector: FileContentSelector, limit: int | None = None
+    ) -> Iterable[Page[MetadataWithFilePath]]:
         if not isinstance(selector, FileIdentifierSelector):
             raise ToolkitNotImplementedError(
                 f"Download with the manifest, {type(selector).__name__}, is not supported for FileContentIO"
@@ -77,7 +91,7 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
             if metadata is None:
                 continue
             identifiers_map = self._as_metadata_map(metadata)
-            downloaded_files: list[FileMetadata] = []
+            downloaded_files: list[MetadataWithFilePath] = []
             for identifier in identifiers:
                 if identifier not in identifiers_map:
                     continue
@@ -94,7 +108,12 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
                     with filepath.open(mode="wb") as file_stream:
                         for chunk in response.iter_bytes(chunk_size=8192):
                             file_stream.write(chunk)
-                downloaded_files.append(meta)
+                downloaded_files.append(
+                    MetadataWithFilePath(
+                        metadata=meta,
+                        file_path=filepath.relative_to(self._target_dir),
+                    )
+                )
             yield Page(items=downloaded_files, worker_id="Main")
 
     def _retrieve_metadata(self, identifiers: Sequence[FileIdentifier]) -> Sequence[FileMetadata] | None:
@@ -142,13 +161,13 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
 
     def _create_filepath(self, meta: FileMetadata, selector: FileIdentifierSelector) -> Path:
         # We now that metadata always have name set
-        filename = Path(cast(str, meta.name))
+        filename = Path(sanitize_filename(cast(str, meta.name)))
         if len(filename.suffix) == 0 and meta.mime_type:
             if mime_ext := mimetypes.guess_extension(meta.mime_type):
                 filename = filename.with_suffix(mime_ext)
-        directory = selector.file_directory
+        directory = sanitize_filename(selector.file_directory)
         if isinstance(meta.directory, str) and meta.directory != "":
-            directory = Path(meta.directory.removeprefix("/"))
+            directory = sanitize_filename(meta.directory.removeprefix("/"))
 
         counter = 1
         filepath = self._target_dir / directory / filename
@@ -185,7 +204,7 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
         return None
 
     def data_to_json_chunk(
-        self, data_chunk: Sequence[FileMetadata], selector: FileContentSelector | None = None
+        self, data_chunk: Sequence[MetadataWithFilePath], selector: FileContentSelector | None = None
     ) -> list[dict[str, JsonVal]]:
         """Convert a writable Cognite resource list to a JSON-compatible chunk of data.
 
@@ -197,7 +216,8 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
         """
         result: list[dict[str, JsonVal]] = []
         for item in data_chunk:
-            item_json = self._crud.dump_resource(item)
+            item_json = self._crud.dump_resource(item.metadata)
+            item_json[FILEPATH] = item.file_path.as_posix()
             result.append(item_json)
         return result
 
@@ -213,7 +233,7 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
         result: list[UploadFileContentItem] = []
         for source_id, item_json in data_chunk:
             item = self.json_to_resource(item_json)
-            filepath = cast(Path, item_json[FILEPATH])
+            filepath = Path(cast(str | Path, item_json[FILEPATH]))
             mime_type, _ = mimetypes.guess_type(filepath)
             # application/octet-stream is the standard fallback for binary data when the type is unknown. (at least Claude thinks so)
             result.append(
@@ -236,7 +256,7 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
         selector: FileContentSelector | None = None,
     ) -> Sequence[HTTPMessage]:
         results: MutableSequence[HTTPMessage] = []
-        if isinstance(selector, FileMetadataTemplateSelector):
+        if isinstance(selector, FileMetadataTemplateSelector | FileIdentifierSelector):
             upload_url_getter = self._upload_url_asset_centric
         elif isinstance(selector, FileDataModelingTemplateSelector):
             upload_url_getter = self._upload_url_data_modeling
@@ -379,13 +399,35 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
     def read_chunks(
         cls, reader: MultiFileReader, selector: FileContentSelector
     ) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
-        for chunk in chunker_sequence(reader.input_files, cls.CHUNK_SIZE):
-            batch: list[tuple[str, dict[str, JsonVal]]] = []
-            for file_path in chunk:
-                metadata = selector.create_instance(file_path)
-                metadata[FILEPATH] = file_path
-                batch.append((str(file_path), metadata))
-            yield batch
+        if isinstance(selector, FileTemplateSelector):
+            for chunk in chunker_sequence(reader.input_files, cls.CHUNK_SIZE):
+                batch: list[tuple[str, dict[str, JsonVal]]] = []
+                for file_path in chunk:
+                    metadata = selector.create_instance(file_path)
+                    metadata[FILEPATH] = file_path
+                    batch.append((file_path.as_posix(), metadata))
+                yield batch
+        elif isinstance(selector, FileIdentifierSelector):
+            for item_chunk in chunker(reader.read_chunks(), cls.CHUNK_SIZE):
+                batch = []
+                for item in item_chunk:
+                    if FILEPATH not in item:
+                        # Todo Log warning
+                        continue
+                    try:
+                        file_path = Path(item[FILEPATH])
+                    except KeyError:
+                        # Todo Log warning
+                        continue
+                    if not file_path.is_absolute():
+                        file_path = reader.input_file.parent / file_path
+                    item[FILEPATH] = file_path
+                    batch.append((file_path.as_posix(), item))
+                yield batch
+        else:
+            raise ToolkitNotImplementedError(
+                f"Reading with the manifest, {type(selector).__name__}, is not supported for FileContentIO"
+            )
 
     @classmethod
     def count_chunks(cls, reader: MultiFileReader) -> int:

cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py

@@ -6,8 +6,6 @@ from typing import Annotated, Any, Literal
 
 from pydantic import ConfigDict, Field, field_validator, model_validator
 
-from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
-
 from ._base import DataSelector, SelectorObject
 from ._instances import SelectedView
 
@@ -17,6 +15,9 @@ FILEPATH = "$FILEPATH"
 
 class FileContentSelector(DataSelector, ABC):
     kind: Literal["FileContent"] = "FileContent"
+
+
+class FileTemplateSelector(FileContentSelector, ABC):
     file_directory: Path
 
     def find_data_files(self, input_dir: Path, manifest_file: Path) -> list[Path]:
@@ -52,7 +53,7 @@ class FileMetadataTemplate(FileTemplate):
         return v
 
 
-class FileMetadataTemplateSelector(FileContentSelector):
+class FileMetadataTemplateSelector(FileTemplateSelector):
     type: Literal["fileMetadataTemplate"] = "fileMetadataTemplate"
     template: FileMetadataTemplate
 
@@ -96,7 +97,7 @@ class FileDataModelingTemplate(FileTemplate):
         return data
 
 
-class FileDataModelingTemplateSelector(FileContentSelector):
+class FileDataModelingTemplateSelector(FileTemplateSelector):
     type: Literal["fileDataModelingTemplate"] = "fileDataModelingTemplate"
     view_id: SelectedView = SelectedView(space="cdf_cdm", external_id="CogniteFile", version="v1")
     template: FileDataModelingTemplate
@@ -150,7 +151,7 @@ FileIdentifier = Annotated[FileInstanceID | FileExternalID | FileInternalID, Fie
 
 class FileIdentifierSelector(FileContentSelector):
     type: Literal["fileIdentifier"] = "fileIdentifier"
-    file_directory: Path = Path("file_content")
+    file_directory: str = "file_content"
     use_metadata_directory: bool = True
     identifiers: tuple[FileIdentifier, ...]
 
@@ -161,6 +162,3 @@ class FileIdentifierSelector(FileContentSelector):
     def __str__(self) -> str:
         hash_ = hashlib.md5(",".join(sorted(str(self.identifiers))).encode()).hexdigest()[:8]
         return f"file_{len(self.identifiers)}_identifiers_{hash_}"
-
-    def create_instance(self, filepath: Path) -> dict[str, Any]:
-        raise ToolkitNotImplementedError("FileIdentifierSelector does not support creating instances from file paths.")

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.20
+      image: cognite/toolkit:0.7.21
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.20
+      image: cognite/toolkit:0.7.21
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.20"
+version = "0.7.21"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.20"
+__version__ = "0.7.21"

{cognite_toolkit-0.7.20.dist-info → cognite_toolkit-0.7.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.20
+Version: 0.7.21
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.7.20.dist-info → cognite_toolkit-0.7.21.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=sefGD2JQuOTBZhEqSj_ECbNZ7nTRN4AwGwX1pSUhoow,5636
-cognite_toolkit/_version.py,sha256=k_NYgI0xLUB_dPJL4N6BdDmPFD0V9fwqtEqrNG-_EGQ,23
+cognite_toolkit/_version.py,sha256=dsLz3w-bEQrCd7hoih-4i3AFeJ0J6O2MZqka_Yl9j0A,23
 cognite_toolkit/config.dev.yaml,sha256=M33FiIKdS3XKif-9vXniQ444GTZ-bLXV8aFH86u9iUQ,332
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
@@ -152,7 +152,7 @@ cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py,sha256=plVGY-hvT0
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py,sha256=U0ItuoNr1KEtoFQAiIe-K19_72ht9-kGndFVgF-iC10,9524
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=SagiSp3JERgEU3SnkjQ76vrxSM7gRA17lvoh0BW4KeQ,64867
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py,sha256=a2HywkruYNJGLZxqOjlp8mrpRGtJDPqIb6qY00eUbEI,17701
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py,sha256=2My16O11WrWomjanLalspZqlQBWG3H-ke2-MTG25bfI,20790
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py,sha256=Tzrkfp9mtseUgnMdpVy_iBTup8S_1caypR5tknedeH4,20791
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py,sha256=vyeRsiIOEbUeYslBsgXoyCk5hozDsubUilA7bdjqS5c,14855
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py,sha256=v3kjn3igwTF53LJ6pp0O8d4S1XFJ1eXQGCchWAcaAx0,28439
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/group_scoped.py,sha256=WEg8-CxMP64WfE_XXIlH114zM51K0uLaYa4atd992zI,1690
@@ -249,7 +249,7 @@ cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=TirKLSNPoLqKjczsw0djW
 cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=ElvqhIEBnhcz0yY1Ds164wVN0_7CFNK-uT0-z7LcR9U,13067
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
 cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=xE1YgoP98-mJjIeF5536KwChzhVY90KYl-bW5sRVhFQ,20206
-cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=qcJDk7YGUZ7YmVTYUDAi8eOK2sozEoxmehpmWrA45Ak,17127
+cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=Wo31GekIP2VZAUpUy-f0qW2b4IjkRo7aiTXIZxbHoFo,18965
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
 cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=pgZN5MbqDwMZl9Ow1KouDJUO2Ngga8_b6hwv7H31SVQ,5161
 cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=VUK1A76zsu4a25A3oaPUrQEEuRcCpUBK6o8UMMKw7qg,2458
@@ -258,7 +258,7 @@ cognite_toolkit/_cdf_tk/storageio/selectors/_base.py,sha256=hjFkbmNGsK3QIW-jnJV_
 cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py,sha256=E9S-wr-JUqRosI_2cSCfR0tF8MdIFTrMxDItuWRcuO4,597
 cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py,sha256=lQHuNtF3i6SEIMPAlziMm0QlqRcvZJ7MKIug6HMTDrs,1012
 cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py,sha256=qdR9wttPUoHZIRQjt2RiLO0cPH8C4CD09GsH1KA5KF4,2343
-cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py,sha256=A2ikNImKTC-WuG5c-WLEJ6LfaB7FytXS57-D2ORuA1k,5326
+cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py,sha256=e7riknOinuhJszkROHeg7iv3foiVz7mIJmva6lMTiOs,5116
 cognite_toolkit/_cdf_tk/storageio/selectors/_instances.py,sha256=NCFSJrAw52bNX6UTfOali8PvNjlqHnvxzL0hYBr7ZmA,4934
 cognite_toolkit/_cdf_tk/storageio/selectors/_raw.py,sha256=sZq9C4G9DMe3S46_usKet0FphQ6ow7cWM_PfXrEAakk,503
 cognite_toolkit/_cdf_tk/tk_warnings/__init__.py,sha256=U9bT-G2xKrX6mmtZ7nZ1FfQeCjNKfKP_p7pev90dwOE,2316
@@ -305,13 +305,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=sVWo0P7uvKyz1D1mrYhdiRak7DOy0o8GxTNtAR3tNrg,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=nLRRf5K6lur4MsjQPl3wSLMTFb4vAf8MBeAmq8bFBok,2430
-cognite_toolkit/_resources/cdf.toml,sha256=V0xczqShauGTujzM49gbP9aLYGTWf4SWo7ZDWDWkxEM,475
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=7RefWun5puw2176WrJSLEVlc9W2-3RR4hZoUZbCamn8,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=KcvP5c2MmkhPJj4kDXu8xRIH98Iyj-VOTzb7gdSMoJ8,2430
+cognite_toolkit/_resources/cdf.toml,sha256=bWCeSoZrULc_3jDfIFKhHbtEmAu5Yt7tuSrGS2LeEkg,475
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.7.20.dist-info/METADATA,sha256=ouWrKIy9SwM_KVNSweTz9FxFNOmODdelIly77qGRhy0,4501
-cognite_toolkit-0.7.20.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cognite_toolkit-0.7.20.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.7.20.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.7.20.dist-info/RECORD,,
+cognite_toolkit-0.7.21.dist-info/METADATA,sha256=_5E4GZxcoVkuzuabVj4_txqit-4Sl4YD62ORO8PkI8U,4501
+cognite_toolkit-0.7.21.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cognite_toolkit-0.7.21.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.7.21.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.7.21.dist-info/RECORD,,