cognite-toolkit 0.6.111__py3-none-any.whl → 0.6.112__py3-none-any.whl

cognite_toolkit/_cdf_tk/storageio/__init__.py

@@ -1,7 +1,4 @@
-from pathlib import Path
-
 from cognite_toolkit._cdf_tk.utils._auxiliary import get_concrete_subclasses
-from cognite_toolkit._cdf_tk.utils.fileio import COMPRESSION_BY_SUFFIX
 
 from ._annotations import AnnotationIO
 from ._applications import CanvasIO, ChartIO
@@ -25,6 +22,7 @@ from ._base import (
 )
 from ._data_classes import InstanceIdCSVList, InstanceIdRow, ModelList
 from ._datapoints import DatapointsIO
+from ._file_content import FileContentIO
 from ._instances import InstanceIO
 from ._raw import RawIO
 from .selectors._base import DataSelector
@@ -34,24 +32,14 @@ STORAGE_IO_CLASSES = get_concrete_subclasses(StorageIO)  # type: ignore[type-abs
 UPLOAD_IO_CLASSES = get_concrete_subclasses(UploadableStorageIO)  # type: ignore[type-abstract]
 
 
-def get_upload_io(selector_cls: type[DataSelector], kind: str | Path) -> type[UploadableStorageIO]:
+def get_upload_io(selector_cls: type[DataSelector]) -> type[UploadableStorageIO]:
     """Get the appropriate UploadableStorageIO class based on the type of the provided selector."""
     for cls in UPLOAD_IO_CLASSES:
-        if issubclass(selector_cls, cls.BASE_SELECTOR) and are_same_kind(cls.KIND, kind):
+        if issubclass(selector_cls, cls.BASE_SELECTOR):
             return cls
     raise ValueError(f"No UploadableStorageIO found for selector of type {selector_cls.__name__}")
 
 
-def are_same_kind(kind: str, kind_or_path: str | Path, /) -> bool:
-    """Check if two kinds are the same, ignoring case and compression suffixes."""
-    if not isinstance(kind_or_path, Path):
-        return kind.casefold() == kind_or_path.casefold()
-    stem = kind_or_path.stem
-    if kind_or_path.suffix in COMPRESSION_BY_SUFFIX:
-        stem = Path(stem).stem
-    return stem.lower().endswith(kind.casefold())
-
-
 __all__ = [
     "AnnotationIO",
     "AssetIO",
@@ -61,6 +49,7 @@ __all__ = [
     "ConfigurableStorageIO",
     "DatapointsIO",
     "EventIO",
+    "FileContentIO",
     "FileMetadataIO",
     "HierarchyIO",
     "InstanceIO",
@@ -76,6 +65,5 @@ __all__ = [
     "TimeSeriesIO",
     "UploadItem",
     "UploadableStorageIO",
-    "are_same_kind",
     "get_upload_io",
 ]

cognite_toolkit/_cdf_tk/storageio/_asset_centric.py

@@ -51,15 +51,9 @@ from cognite_toolkit._cdf_tk.utils.cdf import metadata_key_counts
 from cognite_toolkit._cdf_tk.utils.fileio import FileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.fileio._readers import TableReader
 from cognite_toolkit._cdf_tk.utils.http_client import (
-    FailedRequestItems,
-    FailedRequestMessage,
-    FailedResponse,
-    FailedResponseItems,
     HTTPClient,
     HTTPMessage,
     SimpleBodyRequest,
-    SuccessResponse,
-    SuccessResponseItems,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import (
     T_ID,
@@ -347,7 +341,9 @@ class AssetIO(BaseAssetCentricIO[str, AssetWrite, Asset, AssetWriteList, AssetLi
         return self.client.assets.retrieve_multiple(ids)
 
     @classmethod
-    def read_chunks(cls, reader: FileReader) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
+    def read_chunks(
+        cls, reader: FileReader, selector: AssetCentricSelector
+    ) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
         """Assets require special handling when reading data to ensure parent assets are created first."""
         current_depth = max_depth = 0
         data_name = "row" if isinstance(reader, TableReader) else "line"
@@ -460,22 +456,7 @@ class FileMetadataIO(BaseAssetCentricIO[str, FileMetadataWrite, FileMetadata, Fi
                     body_content=item.dump(),  # type: ignore[arg-type]
                 )
             )
-            # Convert the responses to per-item responses
-            for message in responses:
-                if isinstance(message, SuccessResponse):
-                    results.append(
-                        SuccessResponseItems(status_code=message.status_code, ids=[item.as_id()], body=message.body)
-                    )
-                elif isinstance(message, FailedResponse):
-                    results.append(
-                        FailedResponseItems(
-                            status_code=message.status_code, ids=[item.as_id()], body=message.body, error=message.error
-                        )
-                    )
-                elif isinstance(message, FailedRequestMessage):
-                    results.append(FailedRequestItems(ids=[item.as_id()], error=message.error))
-                else:
-                    results.append(message)
+            results.extend(responses.as_item_responses(item.as_id()))
         return results
 
     def retrieve(self, ids: Sequence[int]) -> FileMetadataList:

cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -216,7 +216,9 @@ class UploadableStorageIO(
         raise NotImplementedError()
 
     @classmethod
-    def read_chunks(cls, reader: MultiFileReader) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
+    def read_chunks(
+        cls, reader: MultiFileReader, selector: T_Selector
+    ) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
         data_name = "row" if reader.is_table else "line"
         # Include name of line for better error messages
         iterable = ((f"{data_name} {line_no}", item) for line_no, item in reader.read_chunks_with_line_numbers())

cognite_toolkit/_cdf_tk/storageio/_datapoints.py

@@ -164,7 +164,9 @@ class DatapointsIO(TableUploadableStorageIO[DataPointsFileSelector, DataPointLis
         )
 
     @classmethod
-    def read_chunks(cls, reader: MultiFileReader) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
+    def read_chunks(
+        cls, reader: MultiFileReader, selector: DataPointsFileSelector
+    ) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
         if not reader.is_table:
             raise RuntimeError("DatapointsIO can only read from TableReader instances.")
         iterator = iter(reader.read_chunks_with_line_numbers())

cognite_toolkit/_cdf_tk/storageio/_file_content.py

@@ -0,0 +1,149 @@
+import json
+import mimetypes
+from collections.abc import Iterable, MutableSequence, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+from cognite.client.data_classes import FileMetadata, FileMetadataWrite
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.cruds import FileMetadataCRUD
+from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
+from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
+from cognite_toolkit._cdf_tk.utils.http_client import (
+    DataBodyRequest,
+    ErrorDetails,
+    FailedResponseItems,
+    HTTPClient,
+    HTTPMessage,
+    SimpleBodyRequest,
+)
+from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
+
+from ._base import Page, UploadableStorageIO, UploadItem
+from .selectors import FileContentSelector, FileMetadataTemplateSelector
+from .selectors._file_content import FILEPATH
+
+
+@dataclass
+class UploadFileContentItem(UploadItem[FileMetadataWrite]):
+    file_path: Path
+    mime_type: str
+
+
+class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileMetadataWrite]):
+    SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
+    SUPPORTED_COMPRESSIONS = frozenset({".gz"})
+    CHUNK_SIZE = 10
+    BASE_SELECTOR = FileContentSelector
+    KIND = "FileContent"
+    SUPPORTED_READ_FORMATS = frozenset({".ndjson"})
+    UPLOAD_ENDPOINT = "/files"
+
+    def __init__(self, client: ToolkitClient) -> None:
+        super().__init__(client)
+        self._crud = FileMetadataCRUD(client, None, None)
+
+    def as_id(self, item: FileMetadata) -> str:
+        return item.external_id or str(item.id)
+
+    def stream_data(self, selector: FileContentSelector, limit: int | None = None) -> Iterable[Page]:
+        raise NotImplementedError("Download of FileContent is not yet supported")
+
+    def count(self, selector: FileContentSelector) -> int | None:
+        return None
+
+    def data_to_json_chunk(
+        self, data_chunk: Sequence[FileMetadata], selector: FileContentSelector | None = None
+    ) -> list[dict[str, JsonVal]]:
+        raise NotImplementedError("Download of FileContent is not yet supported")
+
+    def json_chunk_to_data(self, data_chunk: list[tuple[str, dict[str, JsonVal]]]) -> Sequence[UploadFileContentItem]:
+        """Convert a JSON-compatible chunk of data back to a writable Cognite resource list.
+
+        Args:
+            data_chunk: A list of tuples, each containing a source ID and a dictionary representing
+                the data in a JSON-compatible format.
+        Returns:
+            A writable Cognite resource list representing the data.
+        """
+        result: list[UploadFileContentItem] = []
+        for source_id, item_json in data_chunk:
+            item = self.json_to_resource(item_json)
+            filepath = cast(Path, item_json[FILEPATH])
+            mime_type, _ = mimetypes.guess_type(filepath)
+            # application/octet-stream is the standard fallback for binary data when the type is unknown. (at least Claude thinks so)
+            result.append(
+                UploadFileContentItem(
+                    source_id=source_id,
+                    item=item,
+                    file_path=filepath,
+                    mime_type=mime_type or "application/octet-stream",
+                )
+            )
+        return result
+
+    def json_to_resource(self, item_json: dict[str, JsonVal]) -> FileMetadataWrite:
+        return self._crud.load_resource(item_json)
+
+    def upload_items(
+        self,
+        data_chunk: Sequence[UploadItem[FileMetadataWrite]],
+        http_client: HTTPClient,
+        selector: FileContentSelector | None = None,
+    ) -> Sequence[HTTPMessage]:
+        if not isinstance(selector, FileMetadataTemplateSelector):
+            raise ToolkitNotImplementedError("Only uploading of file metadata is currently supported.")
+        config = http_client.config
+        results: MutableSequence[HTTPMessage] = []
+        for item in cast(Sequence[UploadFileContentItem], data_chunk):
+            responses = http_client.request_with_retries(
+                message=SimpleBodyRequest(
+                    endpoint_url=config.create_api_url(self.UPLOAD_ENDPOINT),
+                    method="POST",
+                    # MyPy does not understand that .dump is valid json
+                    body_content=item.dump(),  # type: ignore[arg-type]
+                )
+            )
+            try:
+                body = responses.get_first_body()
+            except ValueError:
+                results.extend(responses.as_item_responses(item.as_id()))
+                continue
+            try:
+                upload_url = cast(str, body["uploadUrl"])
+            except (KeyError, IndexError):
+                results.append(
+                    FailedResponseItems(
+                        status_code=200,
+                        body=json.dumps(body),
+                        error=ErrorDetails(code=200, message="Malformed response"),
+                        ids=[item.as_id()],
+                    )
+                )
+                continue
+
+            upload_response = http_client.request_with_retries(
+                message=DataBodyRequest(
+                    endpoint_url=upload_url,
+                    method="PUT",
+                    content_type=item.mime_type,
+                    data_content=item.file_path.read_bytes(),
+                )
+            )
+            results.extend(upload_response.as_item_responses(item.as_id()))
+        return results
+
+    @classmethod
+    def read_chunks(
+        cls, reader: MultiFileReader, selector: FileContentSelector
+    ) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
+        for chunk in chunker_sequence(reader.input_files, cls.CHUNK_SIZE):
+            batch: list[tuple[str, dict[str, JsonVal]]] = []
+            for file_path in chunk:
+                metadata = selector.create_instance(file_path)
+                metadata[FILEPATH] = file_path
+                batch.append((str(file_path), metadata))
+            yield batch

cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py

@@ -13,6 +13,12 @@ from ._datapoints import (
     InternalIdColumn,
     TimeSeriesColumn,
 )
+from ._file_content import (
+    FileContentSelector,
+    FileDataModelingTemplateSelector,
+    FileMetadataTemplate,
+    FileMetadataTemplateSelector,
+)
 from ._instances import (
     InstanceFileSelector,
     InstanceSelector,
@@ -33,7 +39,9 @@ Selector = Annotated[
     | AssetCentricFileSelector
     | DataSetSelector
     | DataPointsFileSelector
-    | ChartExternalIdSelector,
+    | ChartExternalIdSelector
+    | FileMetadataTemplateSelector
+    | FileDataModelingTemplateSelector,
     Field(discriminator="type"),
 ]
 
@@ -53,6 +61,10 @@ __all__ = [
     "DataSelector",
     "DataSetSelector",
     "ExternalIdColumn",
+    "FileContentSelector",
+    "FileDataModelingTemplateSelector",
+    "FileMetadataTemplate",
+    "FileMetadataTemplateSelector",
     "InstanceColumn",
     "InstanceFileSelector",
     "InstanceSelector",

cognite_toolkit/_cdf_tk/storageio/selectors/_base.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from pydantic import BaseModel, ConfigDict
 from pydantic.alias_generators import to_camel
 
-from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM
+from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_SUFFIX
 from cognite_toolkit._cdf_tk.utils.file import safe_write, sanitize_filename, yaml_safe_dump
 from cognite_toolkit._cdf_tk.utils.text import to_sentence_case
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
@@ -41,7 +41,7 @@ class DataSelector(SelectorObject, ABC):
             directory: The directory where the YAML file will be saved.
         """
 
-        filepath = directory / f"{sanitize_filename(str(self))}.{DATA_MANIFEST_STEM}.yaml"
+        filepath = directory / f"{sanitize_filename(str(self))}{DATA_MANIFEST_SUFFIX}"
         filepath.parent.mkdir(parents=True, exist_ok=True)
         safe_write(file=filepath, content=yaml_safe_dump(self.model_dump(mode="json", by_alias=True)), encoding="utf-8")
         return filepath
@@ -66,3 +66,15 @@ class DataSelector(SelectorObject, ABC):
     def __str__(self) -> str:
         # We want to force subclasses to implement __str__
         raise NotImplementedError()
+
+    def find_data_files(self, input_dir: Path, manifest_file: Path) -> list[Path]:
+        """Find data files in the specified input directory that match this selector.
+
+        Args:
+            input_dir: The directory to search for data files.
+            manifest_file: The manifest file that describes the data files.
+        Returns:
+            A list of Paths to the data files that match this selector.
+        """
+        data_file_prefix = manifest_file.name.removesuffix(DATA_MANIFEST_SUFFIX)
+        return [file for file in input_dir.glob(f"{data_file_prefix}*") if not file.name.endswith(DATA_MANIFEST_SUFFIX)]

cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py

@@ -0,0 +1,95 @@
+import json
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import ConfigDict, field_validator
+
+from ._base import DataSelector, SelectorObject
+from ._instances import SelectedView
+
+FILENAME_VARIABLE = "$FILENAME"
+FILEPATH = "$FILEPATH"
+
+
+class FileContentSelector(DataSelector, ABC):
+    kind: Literal["FileContent"] = "FileContent"
+    file_directory: Path
+
+    def find_data_files(self, input_dir: Path, manifest_file: Path) -> list[Path]:
+        file_dir = input_dir / self.file_directory
+        if not file_dir.is_dir():
+            return []
+        return [file for file in file_dir.iterdir() if file.is_file()]
+
+    @abstractmethod
+    def create_instance(self, filepath: Path) -> dict[str, Any]: ...
+
+
+class FileTemplate(SelectorObject):
+    model_config = ConfigDict(extra="allow")
+
+    def create_instance(self, filename: str) -> dict[str, Any]:
+        json_str = self.model_dump_json(by_alias=True)
+        return json.loads(json_str.replace(FILENAME_VARIABLE, filename))
+
+
+class FileMetadataTemplate(FileTemplate):
+    name: str
+    external_id: str
+
+    @field_validator("name", "external_id")
+    @classmethod
+    def _validate_filename_in_fields(cls, v: str) -> str:
+        if FILENAME_VARIABLE not in v:
+            raise ValueError(
+                f"{FILENAME_VARIABLE!s} must be present in 'name' and 'external_id' fields. "
+                f"This allows for dynamic substitution based on the file name."
+            )
+        return v
+
+
+class FileMetadataTemplateSelector(FileContentSelector):
+    type: Literal["fileMetadataTemplate"] = "fileMetadataTemplate"
+    template: FileMetadataTemplate
+
+    @property
+    def group(self) -> str:
+        return "FileMetadata"
+
+    def __str__(self) -> str:
+        return "metadata_template"
+
+    def create_instance(self, filepath: Path) -> dict[str, Any]:
+        return self.template.create_instance(filepath.name)
+
+
+class FileDataModelingTemplate(FileTemplate):
+    space: str
+    external_id: str
+
+    @field_validator("external_id")
+    @classmethod
+    def _validate_filename_in_fields(cls, v: str) -> str:
+        if FILENAME_VARIABLE not in v:
+            raise ValueError(
+                f"{FILENAME_VARIABLE!s} must be present in 'external_id' field. "
+                f"This allows for dynamic substitution based on the file name."
+            )
+        return v
+
+
+class FileDataModelingTemplateSelector(FileContentSelector):
+    type: Literal["fileDataModelingTemplate"] = "fileDataModelingTemplate"
+    view_id: SelectedView
+    template: FileDataModelingTemplate
+
+    @property
+    def group(self) -> str:
+        return "FileDataModeling"
+
+    def __str__(self) -> str:
+        return "data_modeling_template"
+
+    def create_instance(self, filepath: Path) -> dict[str, Any]:
+        return self.template.create_instance(filepath.name)

cognite_toolkit/_cdf_tk/utils/fileio/_readers.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from collections import Counter, defaultdict
 from collections.abc import Callable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
-from functools import partial
+from functools import cached_property, partial
 from io import TextIOWrapper
 from pathlib import Path
 from typing import Any
@@ -75,17 +75,25 @@ class MultiFileReader(FileReader):
     def __init__(self, input_files: Sequence[Path]) -> None:
         super().__init__(input_file=input_files[0])
         self.input_files = input_files
+
+    @cached_property
+    def reader_class(self) -> type[FileReader]:
+        """Determine the reader class based on the input files."""
         reader_classes = Counter([FileReader.from_filepath(input_file) for input_file in self.input_files])
         if len(reader_classes) > 1:
             raise ToolkitValueError(
                 "All input files must be of the same format. "
                 f"Found formats: {humanize_collection([cls.FORMAT for cls in reader_classes.keys()])}."
             )
-        self.reader_class = reader_classes.most_common(1)[0][0]
+        return reader_classes.most_common(1)[0][0]
 
     @property
     def is_table(self) -> bool:
-        return issubclass(self.reader_class, TableReader)
+        try:
+            return issubclass(self.reader_class, TableReader)
+        except ValueError:
+            # The input files are not a known format, so it is not a table.
+            return False
 
     @property
     def format(self) -> str:

cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from collections import UserList
-from collections.abc import Sequence
+from collections.abc import Hashable, Sequence
 from dataclasses import dataclass, field
 from typing import Generic, Literal, Protocol, TypeAlias, TypeVar
 
@@ -352,6 +352,24 @@ class ResponseList(UserList[ResponseMessage | FailedRequestMessage]):
                 return _json.loads(resp.body)
         raise ValueError("No successful responses with a body found.")
 
+    def as_item_responses(self, item_id: Hashable) -> list[ResponseMessage | FailedRequestMessage]:
+        # Convert the responses to per-item responses
+        results: list[ResponseMessage | FailedRequestMessage] = []
+        for message in self.data:
+            if isinstance(message, SuccessResponse):
+                results.append(SuccessResponseItems(status_code=message.status_code, ids=[item_id], body=message.body))
+            elif isinstance(message, FailedResponse):
+                results.append(
+                    FailedResponseItems(
+                        status_code=message.status_code, ids=[item_id], body=message.body, error=message.error
+                    )
+                )
+            elif isinstance(message, FailedRequestMessage):
+                results.append(FailedRequestItems(ids=[item_id], error=message.error))
+            else:
+                results.append(message)
+        return results
+
 
 def _dump_body(body: dict[str, JsonVal]) -> str:
     try:

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.111
+      image: cognite/toolkit:0.6.112
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.111
+      image: cognite/toolkit:0.6.112
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.111"
+version = "0.6.112"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.111"
+__version__ = "0.6.112"

{cognite_toolkit-0.6.111.dist-info → cognite_toolkit-0.6.112.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.111
+Version: 0.6.112
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases