cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py

@@ -0,0 +1,164 @@
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Annotated, Any, Literal
+
+from pydantic import ConfigDict, Field, field_validator, model_validator
+
+from ._base import DataSelector, SelectorObject
+from ._instances import SelectedView
+
+FILENAME_VARIABLE = "$FILENAME"
+FILEPATH = "$FILEPATH"
+
+
+class FileContentSelector(DataSelector, ABC):
+    kind: Literal["FileContent"] = "FileContent"
+
+
+class FileTemplateSelector(FileContentSelector, ABC):
+    file_directory: Path
+
+    def find_data_files(self, input_dir: Path, manifest_file: Path) -> list[Path]:
+        file_dir = input_dir / self.file_directory
+        if not file_dir.is_dir():
+            return []
+        return [file for file in file_dir.iterdir() if file.is_file()]
+
+    @abstractmethod
+    def create_instance(self, filepath: Path) -> dict[str, Any]: ...
+
+
+class FileTemplate(SelectorObject):
+    model_config = ConfigDict(extra="allow")
+
+    def create_instance(self, filename: str) -> dict[str, Any]:
+        json_str = self.model_dump_json(by_alias=True)
+        return json.loads(json_str.replace(FILENAME_VARIABLE, filename))
+
+
+class FileMetadataTemplate(FileTemplate):
+    name: str
+    external_id: str
+
+    @field_validator("name", "external_id")
+    @classmethod
+    def _validate_filename_in_fields(cls, v: str) -> str:
+        if FILENAME_VARIABLE not in v:
+            raise ValueError(
+                f"{FILENAME_VARIABLE!s} must be present in 'name' and 'external_id' fields. "
+                f"This allows for dynamic substitution based on the file name."
+            )
+        return v
+
+
+class FileMetadataTemplateSelector(FileTemplateSelector):
+    type: Literal["fileMetadataTemplate"] = "fileMetadataTemplate"
+    template: FileMetadataTemplate
+
+    @property
+    def group(self) -> str:
+        return "FileMetadata"
+
+    def __str__(self) -> str:
+        return "metadata_template"
+
+    def create_instance(self, filepath: Path) -> dict[str, Any]:
+        return self.template.create_instance(filepath.name)
+
+
+class TemplateNodeId(SelectorObject):
+    space: str
+    external_id: str
+
+    @field_validator("external_id")
+    @classmethod
+    def _validate_filename_in_fields(cls, v: str) -> str:
+        if FILENAME_VARIABLE not in v:
+            raise ValueError(
+                f"{FILENAME_VARIABLE!s} must be present in 'external_id' field. "
+                f"This allows for dynamic substitution based on the file name."
+            )
+        return v
+
+
+class FileDataModelingTemplate(FileTemplate):
+    instance_id: TemplateNodeId
+    # Name is required for FileMetadata but not for CogniteFiles. This is the same default behavior as in CDF.
+    name: str = "untitled"
+
+    @model_validator(mode="before")
+    def _move_space_external_id(cls, data: dict[str, Any]) -> dict[str, Any]:
+        if "space" in data and "externalId" in data:
+            data["instanceId"] = {"space": data.pop("space"), "externalId": data.pop("externalId")}
+        elif "space" in data and "external_id" in data:
+            data["instance_id"] = {"space": data.pop("space"), "external_id": data.pop("external_id")}
+        return data
+
+
+class FileDataModelingTemplateSelector(FileTemplateSelector):
+    type: Literal["fileDataModelingTemplate"] = "fileDataModelingTemplate"
+    view_id: SelectedView = SelectedView(space="cdf_cdm", external_id="CogniteFile", version="v1")
+    template: FileDataModelingTemplate
+
+    @property
+    def group(self) -> str:
+        return "FileDataModeling"
+
+    def __str__(self) -> str:
+        return "data_modeling_template"
+
+    def create_instance(self, filepath: Path) -> dict[str, Any]:
+        return self.template.create_instance(filepath.name)
+
+
+class FileIdentifierDefinition(SelectorObject):
+    id_type: str
+
+
+class FileInternalID(FileIdentifierDefinition):
+    id_type: Literal["internalId"] = "internalId"
+    internal_id: int = Field(alias="id")
+
+    def __str__(self) -> str:
+        return f"internalId_{self.internal_id}"
+
+
+class FileExternalID(FileIdentifierDefinition):
+    id_type: Literal["externalId"] = "externalId"
+    external_id: str
+
+    def __str__(self) -> str:
+        return f"externalId_{self.external_id}"
+
+
+class NodeId(SelectorObject):
+    space: str
+    external_id: str
+
+
+class FileInstanceID(FileIdentifierDefinition):
+    id_type: Literal["instanceId"] = "instanceId"
+    instance_id: NodeId
+
+    def __str__(self) -> str:
+        return f"instanceId_{self.instance_id.space}_{self.instance_id.external_id}"
+
+
+FileIdentifier = Annotated[FileInstanceID | FileExternalID | FileInternalID, Field(discriminator="id_type")]
+
+
+class FileIdentifierSelector(FileContentSelector):
+    type: Literal["fileIdentifier"] = "fileIdentifier"
+    file_directory: str = "file_content"
+    use_metadata_directory: bool = True
+    identifiers: tuple[FileIdentifier, ...]
+
+    @property
+    def group(self) -> str:
+        return "Files"
+
+    def __str__(self) -> str:
+        hash_ = hashlib.md5(",".join(sorted(str(self.identifiers))).encode()).hexdigest()[:8]
+        return f"file_{len(self.identifiers)}_identifiers_{hash_}"

cognite_toolkit/_cdf_tk/storageio/selectors/_instances.py

@@ -0,0 +1,162 @@
+from abc import ABC, abstractmethod
+from functools import cached_property
+from pathlib import Path
+from typing import Any, Literal
+
+from cognite.client.data_classes.data_modeling import EdgeId, NodeId, ViewId
+from cognite.client.utils._identifier import InstanceId
+from pydantic import Field
+
+from cognite_toolkit._cdf_tk.constants import DM_EXTERNAL_ID_PATTERN, DM_VERSION_PATTERN, SPACE_FORMAT_PATTERN
+from cognite_toolkit._cdf_tk.storageio._data_classes import InstanceIdCSVList
+
+from ._base import DataSelector, SelectorObject
+
+
+class SelectedView(SelectorObject):
+    space: str = Field(
+        description="Id of the space that the view belongs to.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    external_id: str = Field(
+        description="External-id of the view.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+    )
+    version: str | None = Field(
+        None,
+        description="Version of the view.",
+        max_length=43,
+        pattern=DM_VERSION_PATTERN,
+    )
+
+    def as_id(self) -> ViewId:
+        return ViewId(space=self.space, external_id=self.external_id, version=self.version)
+
+    def __str__(self) -> str:
+        if self.version:
+            return f"{self.space}_{self.external_id}_{self.version}"
+        return f"{self.space}_{self.external_id}"
+
+
+class InstanceSelector(DataSelector, ABC):
+    kind: Literal["Instances"] = "Instances"
+
+    @abstractmethod
+    def get_schema_spaces(self) -> list[str] | None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_instance_spaces(self) -> list[str] | None:
+        raise NotImplementedError()
+
+
+class InstanceSpaceSelector(InstanceSelector):
+    type: Literal["instanceSpace"] = "instanceSpace"
+    instance_space: str
+    instance_type: Literal["node", "edge"] = "node"
+    view: SelectedView | None = None
+
+    def get_schema_spaces(self) -> list[str] | None:
+        return [self.view.space] if self.view else None
+
+    def get_instance_spaces(self) -> list[str] | None:
+        return [self.instance_space]
+
+    @property
+    def group(self) -> str:
+        return self.instance_space
+
+    def __str__(self) -> str:
+        if self.view is None:
+            return self.instance_type
+        return f"{self.view}_{self.instance_type}"
+
+    def as_filter_args(self) -> dict[str, Any]:
+        args: dict[str, Any] = {
+            "instance_type": self.instance_type,
+            "space": self.instance_space,
+        }
+        if self.view:
+            args["source"] = self.view.as_id()
+        return args
+
+
+class InstanceViewSelector(InstanceSelector):
+    type: Literal["instanceView"] = "instanceView"
+    view: SelectedView
+    instance_type: Literal["node", "edge"] = "node"
+    instance_spaces: tuple[str, ...] | None = None
+
+    def get_schema_spaces(self) -> list[str] | None:
+        return [self.view.space]
+
+    def get_instance_spaces(self) -> list[str] | None:
+        return list(self.instance_spaces) if self.instance_spaces else None
+
+    @property
+    def group(self) -> str:
+        return self.view.space
+
+    def __str__(self) -> str:
+        return f"{self.view.external_id}_{self.view.version}_{self.instance_type}"
+
+    def as_filter_args(self) -> dict[str, Any]:
+        args: dict[str, Any] = {
+            "instance_type": self.instance_type,
+            "source": self.view.as_id(),
+        }
+        if self.instance_spaces:
+            args["space"] = list(self.instance_spaces)
+        return args
+
+
+class InstanceFileSelector(InstanceSelector):
+    type: Literal["instanceFile"] = "instanceFile"
+
+    datafile: Path
+    validate_instance: bool = True
+
+    @property
+    def group(self) -> str:
+        return "Instances"
+
+    def __str__(self) -> str:
+        return f"file_{self.datafile.as_posix()}"
+
+    @cached_property
+    def items(self) -> InstanceIdCSVList:
+        return InstanceIdCSVList.read_csv_file(self.datafile)
+
+    @cached_property
+    def _ids_by_type(self) -> tuple[list[NodeId], list[EdgeId]]:
+        node_ids: list[NodeId] = []
+        edge_ids: list[EdgeId] = []
+        for instance in self.items:
+            if instance.instance_type == "node":
+                node_ids.append(NodeId(instance.space, instance.external_id))
+            else:
+                edge_ids.append(EdgeId(instance.space, instance.external_id))
+        return node_ids, edge_ids
+
+    @property
+    def instance_ids(self) -> list[InstanceId]:
+        node_ids, edge_ids = self._ids_by_type
+        return [*node_ids, *edge_ids]
+
+    @property
+    def node_ids(self) -> list[NodeId]:
+        return self._ids_by_type[0]
+
+    @property
+    def edge_ids(self) -> list[EdgeId]:
+        return self._ids_by_type[1]
+
+    def get_schema_spaces(self) -> list[str] | None:
+        return None
+
+    def get_instance_spaces(self) -> list[str] | None:
+        return sorted({instance.space for instance in self.items})

cognite_toolkit/_cdf_tk/storageio/selectors/_raw.py

@@ -0,0 +1,24 @@
+from typing import Literal
+
+from ._base import DataSelector, SelectorObject
+
+
+class SelectedTable(SelectorObject):
+    """Selected RAW table"""
+
+    db_name: str
+    table_name: str
+
+
+class RawTableSelector(DataSelector):
+    type: Literal["rawTable"] = "rawTable"
+    kind: Literal["RawRows"] = "RawRows"
+    table: SelectedTable
+    key: str | None = None
+
+    @property
+    def group(self) -> str:
+        return self.table.db_name
+
+    def __str__(self) -> str:
+        return self.table.table_name

cognite_toolkit/_cdf_tk/storageio/selectors/_three_d.py

@@ -0,0 +1,34 @@
+import hashlib
+from abc import ABC
+from typing import Literal
+
+from ._base import DataSelector
+
+
+class ThreeDSelector(DataSelector, ABC):
+    kind: Literal["3D"] = "3D"
+
+    @property
+    def group(self) -> str:
+        return "3DModels"
+
+
+class ThreeDModelFilteredSelector(ThreeDSelector):
+    type: Literal["3DFiltered"] = "3DFiltered"
+    model_type: Literal["Classic", "DataModel"] = "Classic"
+    published: bool | None = None
+
+    def __str__(self) -> str:
+        suffix = f"3DModels_{self.model_type}"
+        if self.published is not None:
+            return f"{suffix}_published_{self.published}"
+        return suffix
+
+
+class ThreeDModelIdSelector(ThreeDSelector):
+    type: Literal["3DId"] = "3DId"
+    ids: tuple[int, ...]
+
+    def __str__(self) -> str:
+        hash_ = hashlib.md5(",".join(sorted(map(str, self.ids))).encode()).hexdigest()[:8]
+        return f"3DModels_ids_count_{len(self.ids)}_hash_{hash_}"

cognite_toolkit/_cdf_tk/tk_warnings/fileread.py

@@ -215,9 +215,9 @@ class DataSetMissingWarning(IdentifiedResourceFileReadWarning):
 
     def get_message(self) -> str:
         # Avoid circular import
-        from cognite_toolkit._cdf_tk.loaders import TransformationLoader
+        from cognite_toolkit._cdf_tk.cruds import TransformationCRUD
 
-        if self.filepath.parent.name == TransformationLoader.folder_name:
+        if self.filepath.parent.name == TransformationCRUD.folder_name:
             return f"{type(self).__name__}: It is recommended to use a data set if source or destination can be scoped with a data set. If not, ignore this warning."
         else:
             return f"{type(self).__name__}: It is recommended that you set dataSetExternalId for {self.resource_name}. This is missing in {self.filepath.name}. Did you forget to add it?"
@@ -274,9 +274,11 @@ class StreamlitRequirementsWarning(FileReadWarning):
 class ResourceFormatWarning(FileReadWarning):
     severity: ClassVar[SeverityLevel] = SeverityLevel.HIGH
     errors: tuple[str, ...]
+    text: str | None = None
 
     def get_message(self) -> str:
         sep = "\n     * "
         errors = sep.join(map(escape, self.errors))
         s = "s" if len(self.errors) > 1 else ""
-        return f"{type(self).__name__} {len(self.errors)} error{s}:{sep}{errors}"
+        text = self.text or ""
+        return f"{type(self).__name__} {text}{len(self.errors)} error{s}:{sep}{errors}"

cognite_toolkit/_cdf_tk/tk_warnings/other.py

@@ -111,9 +111,13 @@ class MissingDependencyWarning(GeneralWarning):
     dependency_type: str
     identifier: Hashable
     required_by: set[tuple[Hashable, Path]]
+    has_checked_cdf: bool = False
 
     def get_message(self) -> str:
-        msg = f"{self.dependency_type} {self.identifier!r} is missing and is required by:"
+        location = "local configurations"
+        if self.has_checked_cdf:
+            location = "in CDF and local configurations"
+        msg = f"{self.dependency_type} {self.identifier!r} is missing {location} and is required by:"
         for identifier, path in self.required_by:
             msg += f"\n- {identifier!r} in {path}"
         return msg
@@ -132,15 +136,19 @@ class MissingCapabilityWarning(GeneralWarning):
 
 @dataclass(frozen=True)
 class ToolkitDeprecationWarning(ToolkitWarning, DeprecationWarning):
+    severity = SeverityLevel.HIGH
     message: ClassVar[str] = "The '{feature}' is deprecated and will be removed in a future version."
 
     feature: str
     alternative: str | None = None
+    removal_version: str | None = None
 
     def get_message(self) -> str:
         msg = self.message.format(feature=self.feature)
         if self.alternative:
             msg += f"\nUse {self.alternative!r} instead."
+        if self.removal_version:
+            msg += f"\nIt will be removed in version {self.removal_version}."
 
         return msg
 

cognite_toolkit/_cdf_tk/tracker.py

@@ -14,7 +14,7 @@ from mixpanel import Consumer, Mixpanel, MixpanelException
 
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.constants import IN_BROWSER
-from cognite_toolkit._cdf_tk.data_classes._built_modules import BuiltModule
+from cognite_toolkit._cdf_tk.data_classes import CommandTrackingInfo
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
 from cognite_toolkit._cdf_tk.utils import get_cicd_environment
 from cognite_toolkit._version import __version__
@@ -38,13 +38,19 @@ class Tracker:
 
     @property
     def opted_out(self) -> bool:
-        return self._opt_status == "opted-out"
+        return False
 
     @property
     def opted_in(self) -> bool:
-        return self._opt_status == "opted-in"
+        return True
 
-    def track_cli_command(self, warning_list: WarningList[ToolkitWarning], result: str | Exception, cmd: str) -> bool:
+    def track_cli_command(
+        self,
+        warning_list: WarningList[ToolkitWarning],
+        result: str | Exception,
+        cmd: str,
+        additional_tracking_info: CommandTrackingInfo | None = None,
+    ) -> bool:
         warning_count = Counter([type(w).__name__ for w in warning_list])
 
         warning_details: dict[str, str | int] = {}
@@ -52,7 +58,7 @@ class Tracker:
             warning_details[f"warningMostCommon{no}Count"] = count
             warning_details[f"warningMostCommon{no}Name"] = warning
 
-        positional_args, optional_args = self._parse_sys_args()
+        subcommands, optional_args = self._parse_sys_args()
         event_information = {
             "userInput": self.user_command,
             "toolkitVersion": __version__,
@@ -63,23 +69,16 @@ class Tracker:
             **warning_details,
             "result": type(result).__name__ if isinstance(result, Exception) else result,
             "error": str(result) if isinstance(result, Exception) else "",
-            **positional_args,
+            "subcommands": subcommands,
             **optional_args,
-            **{f"alphaFlag-{name}": value for name, value in self._cdf_toml.alpha_flags.items()},
-            **{f"plugin-{name}": value for name, value in self._cdf_toml.plugins.items()},
+            "alphaFlags": [name for name, value in self._cdf_toml.alpha_flags.items() if value],
+            "plugins": [name for name, value in self._cdf_toml.plugins.items() if value],
         }
 
-        return self._track(f"command{cmd.capitalize()}", event_information)
+        if additional_tracking_info:
+            event_information.update(additional_tracking_info.to_dict())
 
-    def track_module_build(self, module: BuiltModule) -> bool:
-        event_information = {
-            "module": module.name,
-            "location_path": module.location.path.as_posix(),
-            "warning_count": module.warning_count,
-            "status": module.status,
-            **{resource_type: len(resource_build) for resource_type, resource_build in module.resources.items()},
-        }
-        return self._track("moduleBuild", event_information)
+        return self._track(f"command{cmd.capitalize()}", event_information)
 
     def _track(self, event_name: str, event_information: dict[str, Any]) -> bool:
         if self.skip_tracking or not self.opted_in or "PYTEST_CURRENT_TEST" in os.environ:
@@ -129,9 +128,9 @@ class Tracker:
         return distinct_id
 
     @staticmethod
-    def _parse_sys_args() -> tuple[dict[str, str], dict[str, str | bool]]:
+    def _parse_sys_args() -> tuple[list[str], dict[str, str | bool]]:
         optional_args: dict[str, str | bool] = {}
-        positional_args: dict[str, str] = {}
+        subcommands: list[str] = []
         last_key: str | None = None
         if sys.argv and len(sys.argv) > 1:
             for arg in sys.argv[1:]:
@@ -148,11 +147,11 @@ class Tracker:
                     optional_args[last_key] = arg
                     last_key = None
                 else:
-                    positional_args[f"positionalArg{len(positional_args)}"] = arg
+                    subcommands.append(arg)
 
             if last_key:
                 optional_args[last_key] = True
-        return positional_args, optional_args
+        return subcommands, optional_args
 
     @property
     def _cicd(self) -> str:

cognite_toolkit/_cdf_tk/utils/__init__.py

@@ -9,9 +9,9 @@ from .file import (
     read_yaml_file,
     safe_read,
     safe_write,
+    sanitize_filename,
     stringify_value_by_key_in_yaml,
     tmp_build_directory,
-    to_directory_compatible,
 )
 from .graphql_parser import GraphQLParser
 from .hashing import (
@@ -45,9 +45,9 @@ __all__ = [
     "safe_read",
     "safe_write",
     "safe_write",
+    "sanitize_filename",
     "sentry_exception_filter",
     "stringify_value_by_key_in_yaml",
     "tmp_build_directory",
     "to_diff",
-    "to_directory_compatible",
 ]

cognite_toolkit/_cdf_tk/utils/_auxiliary.py

@@ -1,14 +1,19 @@
 import inspect
+from abc import ABC
 from typing import TypeVar
 
+from cognite_toolkit import _version
+
 T_Cls = TypeVar("T_Cls")
 
 
-def get_concrete_subclasses(base_cls: type[T_Cls]) -> list[type[T_Cls]]:
+def get_concrete_subclasses(base_cls: type[T_Cls], exclude_ABC_base: bool = True) -> list[type[T_Cls]]:
     """
     Returns a list of all concrete subclasses of the given base class.
     Args:
         base_cls (type[T_Cls]): The base class to find subclasses for.
+        exclude_ABC_base (bool): If True, excludes subclasses of ABC base classes.
+            Defaults to True.
     Returns:
         list[type[T_Cls]]: A list of concrete subclasses of the base class.
     """
@@ -20,8 +25,12 @@ def get_concrete_subclasses(base_cls: type[T_Cls]) -> list[type[T_Cls]]:
         for subclass in current_cls.__subclasses__():
             if subclass in seen:
                 continue
-            if not inspect.isabstract(subclass):
+            if (not inspect.isabstract(subclass)) and (not exclude_ABC_base or ABC not in subclass.__bases__):
                 subclasses.append(subclass)
             seen.add(subclass)
             to_check.append(subclass)
     return subclasses
+
+
+def get_current_toolkit_version() -> str:
+    return _version.__version__