cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/storageio/selectors/_base.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+from pydantic.alias_generators import to_camel
+
+from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_SUFFIX
+from cognite_toolkit._cdf_tk.utils.file import safe_write, sanitize_filename, yaml_safe_dump
+from cognite_toolkit._cdf_tk.utils.text import to_sentence_case
+from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
+
+
+class SelectorObject(BaseModel):
+    """This is used as base class for all selector objects including nested ones."""
+
+    model_config = ConfigDict(frozen=True, alias_generator=to_camel, populate_by_name=True)
+
+
+class DataSelector(SelectorObject, ABC):
+    """A selector gives instructions on what data to select from CDF.
+
+    For example, for instances it can be a view or container, while for assets it can be a data set or asset subtree.
+
+    Args:
+        type: The type of selector.
+        kind: The kind of data the selector is for (e.g., 'RawRows', 'Assets').
+    """
+
+    type: str
+    kind: str
+
+    def dump(self) -> dict[str, JsonVal]:
+        return self.model_dump(by_alias=True)
+
+    def dump_to_file(self, directory: Path) -> Path:
+        """Dumps the selector to a YAML file in the specified directory.
+
+        The filename is derived from the string representation of the selector.
+
+        Args:
+            directory: The directory where the YAML file will be saved.
+        """
+
+        filepath = directory / f"{sanitize_filename(str(self))}{DATA_MANIFEST_SUFFIX}"
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+        safe_write(file=filepath, content=yaml_safe_dump(self.model_dump(mode="json", by_alias=True)), encoding="utf-8")
+        return filepath
+
+    @property
+    @abstractmethod
+    def group(self) -> str:
+        """A string representing the group of the selector, used for organizing files.
+
+        It is used when downloading to determine the subdirectory within the output directory.
+
+        For example, for raw table, the group would be the database name, while the selector itself
+        would be the table name.
+        """
+        raise NotImplementedError()
+
+    @property
+    def display_name(self) -> str:
+        """A human-readable name for the selector."""
+        return to_sentence_case(self.kind)
+
+    def __str__(self) -> str:
+        # We want to force subclasses to implement __str__
+        raise NotImplementedError()
+
+    def find_data_files(self, input_dir: Path, manifest_file: Path) -> list[Path]:
+        """Find data files in the specified input directory that match this selector.
+
+        Args:
+            input_dir: The directory to search for data files.
+            manifest_file: The manifest file that describes the data files.
+        Returns:
+            A list of Paths to the data files that match this selector.
+        """
+        data_file_prefix = manifest_file.name.removesuffix(DATA_MANIFEST_SUFFIX)
+        return [file for file in input_dir.glob(f"{data_file_prefix}*") if not file.name.endswith(DATA_MANIFEST_SUFFIX)]

cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py

@@ -0,0 +1,22 @@
+import hashlib
+from abc import ABC
+from typing import Literal
+
+from ._base import DataSelector
+
+
+class CanvasSelector(DataSelector, ABC):
+    kind: Literal["IndustrialCanvas"] = "IndustrialCanvas"
+
+
+class CanvasExternalIdSelector(CanvasSelector):
+    type: Literal["canvasExternalId"] = "canvasExternalId"
+    external_ids: tuple[str, ...]
+
+    @property
+    def group(self) -> str:
+        return "Canvas"
+
+    def __str__(self) -> str:
+        hash_ = hashlib.md5(",".join(sorted(self.external_ids)).encode()).hexdigest()[:8]
+        return f"canvas_count_{len(self.external_ids)}_hash_{hash_}"

cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py

@@ -0,0 +1,45 @@
+import hashlib
+from abc import ABC
+from typing import Literal
+
+from ._base import DataSelector
+
+
+class ChartSelector(DataSelector, ABC):
+    kind: Literal["Charts"] = "Charts"
+
+
+class ChartOwnerSelector(ChartSelector):
+    type: Literal["chartOwner"] = "chartOwner"
+    owner_id: str
+
+    @property
+    def group(self) -> str:
+        return "Charts"
+
+    def __str__(self) -> str:
+        return self.owner_id
+
+
+class AllChartsSelector(ChartSelector):
+    type: Literal["allCharts"] = "allCharts"
+
+    @property
+    def group(self) -> str:
+        return "Charts"
+
+    def __str__(self) -> str:
+        return "all"
+
+
+class ChartExternalIdSelector(ChartSelector):
+    type: Literal["chartExternalId"] = "chartExternalId"
+    external_ids: tuple[str, ...]
+
+    @property
+    def group(self) -> str:
+        return "Charts"
+
+    def __str__(self) -> str:
+        hash_ = hashlib.md5(",".join(sorted(self.external_ids)).encode()).hexdigest()[:8]
+        return f"chart_count_{len(self.external_ids)}_hash_{hash_}"

cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py

@@ -0,0 +1,89 @@
+from abc import ABC, abstractmethod
+from functools import cached_property
+from typing import Annotated, Any, ClassVar, Literal
+
+from cognite.client._proto.data_points_pb2 import (
+    InstanceId,
+)
+from pydantic import Field
+
+from ._base import DataSelector, SelectorObject
+
+
+class Column(SelectorObject, ABC):
+    column_type: str
+    column: str
+    dtype: Literal["numeric", "string"]
+
+    @abstractmethod
+    def as_wrapped_id(self) -> dict[str, Any]: ...
+
+
+class InstanceColumn(Column):
+    column_type: Literal["instance"] = "instance"
+    space: str
+    external_id: str
+
+    def as_wrapped_id(self) -> dict[str, Any]:
+        return {"instanceId": InstanceId(space=self.space, externalId=self.external_id)}
+
+
+class ExternalIdColumn(Column):
+    column_type: Literal["externalId"] = "externalId"
+    external_id: str
+
+    def as_wrapped_id(self) -> dict[str, Any]:
+        return {"externalId": self.external_id}
+
+
+class InternalIdColumn(Column):
+    column_type: Literal["internalId"] = "internalId"
+    internal_id: int
+
+    def as_wrapped_id(self) -> dict[str, Any]:
+        return {"id": self.internal_id}
+
+
+TimeSeriesColumn = Annotated[
+    InstanceColumn | ExternalIdColumn | InternalIdColumn,
+    Field(discriminator="column_type"),
+]
+
+
+class DataPointsSelector(DataSelector, ABC):
+    kind: Literal["Datapoints"] = "Datapoints"
+
+
+class DataPointsFileSelector(DataPointsSelector):
+    type: Literal["datapointsFile"] = "datapointsFile"
+
+    timestamp_column: str
+    columns: tuple[TimeSeriesColumn, ...]
+
+    @property
+    def group(self) -> str:
+        return "Datapoints"
+
+    def __str__(self) -> str:
+        return "datapoints_file"
+
+    @cached_property
+    def id_by_column(self) -> dict[str, Column]:
+        return {col.column: col for col in self.columns}
+
+
+class DataPointsDataSetSelector(DataPointsSelector):
+    required_columns: ClassVar[frozenset[str]] = frozenset({"externalId", "timestamp", "value"})
+    type: Literal["datapointsDataSet"] = "datapointsDataSet"
+
+    data_set_external_id: str
+    start: int | str | None = None
+    end: int | str | None = None
+    data_type: Literal["numeric", "string"] = "numeric"
+
+    @property
+    def group(self) -> str:
+        return f"DataSet_{self.data_set_external_id}"
+
+    def __str__(self) -> str:
+        return f"datapoints_dataset_{self.data_set_external_id}"

cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py

@@ -0,0 +1,164 @@
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Annotated, Any, Literal
+
+from pydantic import ConfigDict, Field, field_validator, model_validator
+
+from ._base import DataSelector, SelectorObject
+from ._instances import SelectedView
+
+FILENAME_VARIABLE = "$FILENAME"
+FILEPATH = "$FILEPATH"
+
+
+class FileContentSelector(DataSelector, ABC):
+    kind: Literal["FileContent"] = "FileContent"
+
+
+class FileTemplateSelector(FileContentSelector, ABC):
+    file_directory: Path
+
+    def find_data_files(self, input_dir: Path, manifest_file: Path) -> list[Path]:
+        file_dir = input_dir / self.file_directory
+        if not file_dir.is_dir():
+            return []
+        return [file for file in file_dir.iterdir() if file.is_file()]
+
+    @abstractmethod
+    def create_instance(self, filepath: Path) -> dict[str, Any]: ...
+
+
+class FileTemplate(SelectorObject):
+    model_config = ConfigDict(extra="allow")
+
+    def create_instance(self, filename: str) -> dict[str, Any]:
+        json_str = self.model_dump_json(by_alias=True)
+        return json.loads(json_str.replace(FILENAME_VARIABLE, filename))
+
+
+class FileMetadataTemplate(FileTemplate):
+    name: str
+    external_id: str
+
+    @field_validator("name", "external_id")
+    @classmethod
+    def _validate_filename_in_fields(cls, v: str) -> str:
+        if FILENAME_VARIABLE not in v:
+            raise ValueError(
+                f"{FILENAME_VARIABLE!s} must be present in 'name' and 'external_id' fields. "
+                f"This allows for dynamic substitution based on the file name."
+            )
+        return v
+
+
+class FileMetadataTemplateSelector(FileTemplateSelector):
+    type: Literal["fileMetadataTemplate"] = "fileMetadataTemplate"
+    template: FileMetadataTemplate
+
+    @property
+    def group(self) -> str:
+        return "FileMetadata"
+
+    def __str__(self) -> str:
+        return "metadata_template"
+
+    def create_instance(self, filepath: Path) -> dict[str, Any]:
+        return self.template.create_instance(filepath.name)
+
+
+class TemplateNodeId(SelectorObject):
+    space: str
+    external_id: str
+
+    @field_validator("external_id")
+    @classmethod
+    def _validate_filename_in_fields(cls, v: str) -> str:
+        if FILENAME_VARIABLE not in v:
+            raise ValueError(
+                f"{FILENAME_VARIABLE!s} must be present in 'external_id' field. "
+                f"This allows for dynamic substitution based on the file name."
+            )
+        return v
+
+
+class FileDataModelingTemplate(FileTemplate):
+    instance_id: TemplateNodeId
+    # Name is required for FileMetadata but not for CogniteFiles. This is the same default behavior as in CDF.
+    name: str = "untitled"
+
+    @model_validator(mode="before")
+    def _move_space_external_id(cls, data: dict[str, Any]) -> dict[str, Any]:
+        if "space" in data and "externalId" in data:
+            data["instanceId"] = {"space": data.pop("space"), "externalId": data.pop("externalId")}
+        elif "space" in data and "external_id" in data:
+            data["instance_id"] = {"space": data.pop("space"), "external_id": data.pop("external_id")}
+        return data
+
+
+class FileDataModelingTemplateSelector(FileTemplateSelector):
+    type: Literal["fileDataModelingTemplate"] = "fileDataModelingTemplate"
+    view_id: SelectedView = SelectedView(space="cdf_cdm", external_id="CogniteFile", version="v1")
+    template: FileDataModelingTemplate
+
+    @property
+    def group(self) -> str:
+        return "FileDataModeling"
+
+    def __str__(self) -> str:
+        return "data_modeling_template"
+
+    def create_instance(self, filepath: Path) -> dict[str, Any]:
+        return self.template.create_instance(filepath.name)
+
+
+class FileIdentifierDefinition(SelectorObject):
+    id_type: str
+
+
+class FileInternalID(FileIdentifierDefinition):
+    id_type: Literal["internalId"] = "internalId"
+    internal_id: int = Field(alias="id")
+
+    def __str__(self) -> str:
+        return f"internalId_{self.internal_id}"
+
+
+class FileExternalID(FileIdentifierDefinition):
+    id_type: Literal["externalId"] = "externalId"
+    external_id: str
+
+    def __str__(self) -> str:
+        return f"externalId_{self.external_id}"
+
+
+class NodeId(SelectorObject):
+    space: str
+    external_id: str
+
+
+class FileInstanceID(FileIdentifierDefinition):
+    id_type: Literal["instanceId"] = "instanceId"
+    instance_id: NodeId
+
+    def __str__(self) -> str:
+        return f"instanceId_{self.instance_id.space}_{self.instance_id.external_id}"
+
+
+FileIdentifier = Annotated[FileInstanceID | FileExternalID | FileInternalID, Field(discriminator="id_type")]
+
+
+class FileIdentifierSelector(FileContentSelector):
+    type: Literal["fileIdentifier"] = "fileIdentifier"
+    file_directory: str = "file_content"
+    use_metadata_directory: bool = True
+    identifiers: tuple[FileIdentifier, ...]
+
+    @property
+    def group(self) -> str:
+        return "Files"
+
+    def __str__(self) -> str:
+        hash_ = hashlib.md5(",".join(sorted(str(self.identifiers))).encode()).hexdigest()[:8]
+        return f"file_{len(self.identifiers)}_identifiers_{hash_}"

cognite_toolkit/_cdf_tk/storageio/selectors/_instances.py

@@ -0,0 +1,162 @@
+from abc import ABC, abstractmethod
+from functools import cached_property
+from pathlib import Path
+from typing import Any, Literal
+
+from cognite.client.data_classes.data_modeling import EdgeId, NodeId, ViewId
+from cognite.client.utils._identifier import InstanceId
+from pydantic import Field
+
+from cognite_toolkit._cdf_tk.constants import DM_EXTERNAL_ID_PATTERN, DM_VERSION_PATTERN, SPACE_FORMAT_PATTERN
+from cognite_toolkit._cdf_tk.storageio._data_classes import InstanceIdCSVList
+
+from ._base import DataSelector, SelectorObject
+
+
+class SelectedView(SelectorObject):
+    space: str = Field(
+        description="Id of the space that the view belongs to.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    external_id: str = Field(
+        description="External-id of the view.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+    )
+    version: str | None = Field(
+        None,
+        description="Version of the view.",
+        max_length=43,
+        pattern=DM_VERSION_PATTERN,
+    )
+
+    def as_id(self) -> ViewId:
+        return ViewId(space=self.space, external_id=self.external_id, version=self.version)
+
+    def __str__(self) -> str:
+        if self.version:
+            return f"{self.space}_{self.external_id}_{self.version}"
+        return f"{self.space}_{self.external_id}"
+
+
+class InstanceSelector(DataSelector, ABC):
+    kind: Literal["Instances"] = "Instances"
+
+    @abstractmethod
+    def get_schema_spaces(self) -> list[str] | None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_instance_spaces(self) -> list[str] | None:
+        raise NotImplementedError()
+
+
+class InstanceSpaceSelector(InstanceSelector):
+    type: Literal["instanceSpace"] = "instanceSpace"
+    instance_space: str
+    instance_type: Literal["node", "edge"] = "node"
+    view: SelectedView | None = None
+
+    def get_schema_spaces(self) -> list[str] | None:
+        return [self.view.space] if self.view else None
+
+    def get_instance_spaces(self) -> list[str] | None:
+        return [self.instance_space]
+
+    @property
+    def group(self) -> str:
+        return self.instance_space
+
+    def __str__(self) -> str:
+        if self.view is None:
+            return self.instance_type
+        return f"{self.view}_{self.instance_type}"
+
+    def as_filter_args(self) -> dict[str, Any]:
+        args: dict[str, Any] = {
+            "instance_type": self.instance_type,
+            "space": self.instance_space,
+        }
+        if self.view:
+            args["source"] = self.view.as_id()
+        return args
+
+
+class InstanceViewSelector(InstanceSelector):
+    type: Literal["instanceView"] = "instanceView"
+    view: SelectedView
+    instance_type: Literal["node", "edge"] = "node"
+    instance_spaces: tuple[str, ...] | None = None
+
+    def get_schema_spaces(self) -> list[str] | None:
+        return [self.view.space]
+
+    def get_instance_spaces(self) -> list[str] | None:
+        return list(self.instance_spaces) if self.instance_spaces else None
+
+    @property
+    def group(self) -> str:
+        return self.view.space
+
+    def __str__(self) -> str:
+        return f"{self.view.external_id}_{self.view.version}_{self.instance_type}"
+
+    def as_filter_args(self) -> dict[str, Any]:
+        args: dict[str, Any] = {
+            "instance_type": self.instance_type,
+            "source": self.view.as_id(),
+        }
+        if self.instance_spaces:
+            args["space"] = list(self.instance_spaces)
+        return args
+
+
+class InstanceFileSelector(InstanceSelector):
+    type: Literal["instanceFile"] = "instanceFile"
+
+    datafile: Path
+    validate_instance: bool = True
+
+    @property
+    def group(self) -> str:
+        return "Instances"
+
+    def __str__(self) -> str:
+        return f"file_{self.datafile.as_posix()}"
+
+    @cached_property
+    def items(self) -> InstanceIdCSVList:
+        return InstanceIdCSVList.read_csv_file(self.datafile)
+
+    @cached_property
+    def _ids_by_type(self) -> tuple[list[NodeId], list[EdgeId]]:
+        node_ids: list[NodeId] = []
+        edge_ids: list[EdgeId] = []
+        for instance in self.items:
+            if instance.instance_type == "node":
+                node_ids.append(NodeId(instance.space, instance.external_id))
+            else:
+                edge_ids.append(EdgeId(instance.space, instance.external_id))
+        return node_ids, edge_ids
+
+    @property
+    def instance_ids(self) -> list[InstanceId]:
+        node_ids, edge_ids = self._ids_by_type
+        return [*node_ids, *edge_ids]
+
+    @property
+    def node_ids(self) -> list[NodeId]:
+        return self._ids_by_type[0]
+
+    @property
+    def edge_ids(self) -> list[EdgeId]:
+        return self._ids_by_type[1]
+
+    def get_schema_spaces(self) -> list[str] | None:
+        return None
+
+    def get_instance_spaces(self) -> list[str] | None:
+        return sorted({instance.space for instance in self.items})

cognite_toolkit/_cdf_tk/storageio/selectors/_raw.py

@@ -0,0 +1,24 @@
+from typing import Literal
+
+from ._base import DataSelector, SelectorObject
+
+
+class SelectedTable(SelectorObject):
+    """Selected RAW table"""
+
+    db_name: str
+    table_name: str
+
+
+class RawTableSelector(DataSelector):
+    type: Literal["rawTable"] = "rawTable"
+    kind: Literal["RawRows"] = "RawRows"
+    table: SelectedTable
+    key: str | None = None
+
+    @property
+    def group(self) -> str:
+        return self.table.db_name
+
+    def __str__(self) -> str:
+        return self.table.table_name

cognite_toolkit/_cdf_tk/tk_warnings/fileread.py

@@ -274,9 +274,11 @@ class StreamlitRequirementsWarning(FileReadWarning):
 class ResourceFormatWarning(FileReadWarning):
     severity: ClassVar[SeverityLevel] = SeverityLevel.HIGH
     errors: tuple[str, ...]
+    text: str | None = None
 
     def get_message(self) -> str:
         sep = "\n     * "
         errors = sep.join(map(escape, self.errors))
         s = "s" if len(self.errors) > 1 else ""
-        return f"{type(self).__name__} {len(self.errors)} error{s}:{sep}{errors}"
+        text = self.text or ""
+        return f"{type(self).__name__} {text}{len(self.errors)} error{s}:{sep}{errors}"

cognite_toolkit/_cdf_tk/tk_warnings/other.py

@@ -136,15 +136,19 @@ class MissingCapabilityWarning(GeneralWarning):
 
 @dataclass(frozen=True)
 class ToolkitDeprecationWarning(ToolkitWarning, DeprecationWarning):
+    severity = SeverityLevel.HIGH
     message: ClassVar[str] = "The '{feature}' is deprecated and will be removed in a future version."
 
     feature: str
     alternative: str | None = None
+    removal_version: str | None = None
 
     def get_message(self) -> str:
         msg = self.message.format(feature=self.feature)
         if self.alternative:
             msg += f"\nUse {self.alternative!r} instead."
+        if self.removal_version:
+            msg += f"\nIt will be removed in version {self.removal_version}."
 
         return msg