cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.39__py3-none-any.whl

cognite_toolkit/_cdf_tk/resource_classes/infield_cdm_location_config.py

@@ -0,0 +1,109 @@
+from typing import Any
+
+from .base import BaseModelResource, ToolkitResource
+
+
+class FeatureToggles(BaseModelResource):
+    """Feature toggles for InField location configuration."""
+
+    three_d: bool | None = None
+    trends: bool | None = None
+    documents: bool | None = None
+    workorders: bool | None = None
+    notifications: bool | None = None
+    media: bool | None = None
+    template_checklist_flow: bool | None = None
+
+
+class AccessManagement(BaseModelResource):
+    """Access management configuration."""
+
+    template_admins: list[str] | None = None  # list of CDF group external IDs
+    checklist_admins: list[str] | None = None  # list of CDF group external IDs
+
+
+class DataFilterPath(BaseModelResource):
+    """Path reference for data filters."""
+
+    space: str
+    external_id: str
+
+
+class DataFilter(BaseModelResource):
+    """Data filter configuration for a resource type."""
+
+    path: DataFilterPath | None = None
+    instance_spaces: list[str] | None = None
+
+
+class DataFilters(BaseModelResource):
+    """Data filters configuration."""
+
+    files: DataFilter | None = None
+    assets: DataFilter | None = None
+    operations: DataFilter | None = None
+    time_series: DataFilter | None = None
+    notifications: DataFilter | None = None
+    maintenance_orders: DataFilter | None = None
+
+
+class DataStorage(BaseModelResource):
+    """Data storage configuration."""
+
+    root_location: DataFilterPath | None = None
+    app_instance_space: str | None = None
+
+
+class ViewMapping(BaseModelResource):
+    """View mapping configuration."""
+
+    space: str
+    version: str
+    external_id: str
+
+
+class ViewMappings(BaseModelResource):
+    """View mappings configuration."""
+
+    asset: ViewMapping | None = None
+    operation: ViewMapping | None = None
+    notification: ViewMapping | None = None
+    maintenance_order: ViewMapping | None = None
+
+
+class Discipline(BaseModelResource):
+    """Discipline configuration."""
+
+    name: str
+    external_id: str
+
+
+class InFieldCDMLocationConfigYAML(ToolkitResource):
+    """Properties for InFieldCDMLocationConfig node.
+
+    Fields:
+    - space: The space of the InField CDM location configuration
+    - external_id: The external ID of the InField CDM location configuration
+    - name: The name of the location configuration
+    - description: The description of the location configuration
+    - feature_toggles: Feature toggles configuration
+    - access_management: Access management configuration
+    - data_filters: Data filters configuration for various resource types
+    - data_storage: Data storage configuration
+    - view_mappings: View mappings configuration
+    - disciplines: List of disciplines
+    - data_exploration_config: Data exploration configuration
+    """
+
+    space: str
+    external_id: str
+
+    name: str | None = None
+    description: str | None = None
+    feature_toggles: FeatureToggles | None = None
+    access_management: AccessManagement | None = None
+    data_filters: DataFilters | None = None
+    data_storage: DataStorage | None = None
+    view_mappings: ViewMappings | None = None
+    disciplines: list[Discipline] | None = None
+    data_exploration_config: dict[str, Any] | None = None

cognite_toolkit/_cdf_tk/resource_classes/migration.py

@@ -1,19 +1,20 @@
-import re
 from typing import Literal
 
-from pydantic import Field, field_validator
+from pydantic import Field
 
-from cognite_toolkit._cdf_tk.utils import humanize_collection
+from cognite_toolkit._cdf_tk.constants import INSTANCE_EXTERNAL_ID_PATTERN
 
 from .base import ToolkitResource
 from .view_field_definitions import ViewReference
 
-JSON_PATH_PATTERN = re.compile(
-    r"^([a-zA-Z_][a-zA-Z0-9_]*|\[\d+\])(\.[a-zA-Z_][a-zA-Z0-9_]*|\[\d+\]|\[\'[^\']*\'\]|\[\"[^\"]*\"\])*$"
-)
-
 
 class ResourceViewMappingYAML(ToolkitResource):
+    external_id: str = Field(
+        description="External-id of the mapping.",
+        min_length=1,
+        max_length=256,
+        pattern=INSTANCE_EXTERNAL_ID_PATTERN,
+    )
     resource_type: Literal["asset", "event", "file", "timeSeries", "sequence", "assetAnnotation", "fileAnnotation"] = (
         Field(
             description="The type of the resource to map to the view.",
@@ -25,13 +26,3 @@ class ResourceViewMappingYAML(ToolkitResource):
     property_mapping: dict[str, str] = Field(
         description="A dictionary mapping from resource property to view property.",
     )
-
-    @field_validator("property_mapping")
-    @classmethod
-    def validate_json_paths(cls, value: dict[str, str]) -> dict[str, str]:
-        if not isinstance(value, dict):
-            return value
-        not_matching_keys = [k for k in value.keys() if not re.match(JSON_PATH_PATTERN, k)]
-        if not_matching_keys:
-            raise ValueError(f"Invalid JSON paths: {humanize_collection(not_matching_keys)}")
-        return value

cognite_toolkit/_cdf_tk/resource_classes/search_config.py

@@ -2,7 +2,7 @@ from typing import ClassVar
 
 from pydantic import Field
 
-from cognite_toolkit._cdf_tk.client.data_classes.search_config import SearchConfigWrite
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.search_config import SearchConfigWrite
 
 from .base import BaseModelResource, ToolkitResource
 

cognite_toolkit/_cdf_tk/resource_classes/streams.py

@@ -0,0 +1,29 @@
+from typing import Literal
+
+from pydantic import Field
+
+from cognite_toolkit._cdf_tk.constants import StreamTemplateName
+
+from .base import BaseModelResource, ToolkitResource
+
+
+class StreamSettings(BaseModelResource):
+    """Stream settings resource class."""
+
+    template: dict[Literal["name"], StreamTemplateName] = Field(
+        description="Reference to a template which should be used to define initial settings for the stream."
+    )
+
+
+class StreamYAML(ToolkitResource):
+    """Stream YAML resource class."""
+
+    external_id: str = Field(
+        description="The external ID of the stream.",
+        min_length=1,
+        max_length=100,
+        pattern="^[a-z]([a-z0-9_-]{0,98}[a-z0-9])?$",
+    )
+    settings: StreamSettings = Field(
+        description="Stream settings which should be applied to a stream.",
+    )

cognite_toolkit/_cdf_tk/resource_classes/workflow_version.py

@@ -1,4 +1,5 @@
-from typing import Literal
+from abc import ABC
+from typing import Annotated, Literal
 
 from cognite.client.data_classes import WorkflowVersionUpsert
 from pydantic import Field, JsonValue
@@ -6,18 +7,141 @@ from pydantic import Field, JsonValue
 from .base import BaseModelResource, ToolkitResource
 
 
+class WorkflowVersionId(BaseModelResource):
+    workflow_external_id: str = Field(
+        max_length=255,
+        description="Identifier for a workflow. Must be unique for the project. No trailing or leading whitespace and no null characters allowed.",
+    )
+    version: str = Field(
+        max_length=255,
+        description="Identifier for a version. Must be unique for the workflow. No trailing or leading whitespace and no null characters allowed.",
+    )
+
+
+class CogniteFunctionRef(BaseModelResource):
+    external_id: str = Field(
+        description="The external id of the Cognite Function in the project. This can be either a function external ID or a reference like ${myTaskExternalId.output.someKey}"
+    )
+    data: str | JsonValue | None = Field(
+        None, description="Input data that will be passed to the Cognite Function. Limited to 100KB in size."
+    )
+
+
+class FunctionTaskParameters(BaseModelResource):
+    function: CogniteFunctionRef
+    is_async_complete: bool = Field(
+        False, description="Defines if the execution of the task should be completed asynchronously."
+    )
+
+
+class TransformationRef(BaseModelResource):
+    external_id: str = Field(
+        description="The external id of the Transformation in the project. This can be either a transformation external ID or a reference like ${myTaskExternalId.output.someKey}"
+    )
+    concurrency_policy: Literal["fail", "waitForCurrent", "restartAfterCurrent"] = Field(
+        "fail",
+        description="""Determines the behavior of the task if the Transformation is already running.
+
+fail: The task fails if another instance of the Transformation is currently running.
+waitForCurrent: The task will pause and wait for the already running Transformation to complete. Once completed, the task is completed. This mode is useful for preventing redundant Transformation runs.
+restartAfterCurrent: The task waits for the ongoing Transformation to finish. After completion, the task restarts the Transformation. This mode ensures that the most recent data can be used by following tasks.""",
+    )
+    use_transformation_credentials: bool = Field(
+        False,
+        description="If set to true, the transformation will run using the client credentials configured on the transformation. If set to false, the transformation will run using the client credentials used to trigger the workflow.",
+    )
+
+
+class TransformationTaskParameters(BaseModelResource):
+    transformation: TransformationRef
+
+
+class CDFRequest(BaseModelResource):
+    resource_path: str = Field(
+        description="The path of the request. The path should be prefixed by {cluster}.cognitedata.com/api/v1/project/{project} based on the relevant cluster and project. It can also contain references like ${myTaskExternalId.output.someKey}"
+    )
+    method: Literal["POST", "GET", "PUT"] | str = Field(
+        description="The HTTP method of the request. It can also be a reference like ${myTaskExternalId.output.someKey}"
+    )
+    query_parameters: dict[str, JsonValue] | str | None = Field(
+        None,
+        description="The query parameters of the request. It can also be a reference like ${myTaskExternalId.output.someKey}",
+    )
+    body: JsonValue | str | None = Field(
+        None, description="The body of the request. It can also be a reference like ${myTaskExternalId.output.someKey}"
+    )
+    request_timeout_in_millis: float | str | None = Field(
+        None,
+        description="The timeout for the request in milliseconds. It can also be a reference like ${myTaskExternalId.output.someKey}",
+    )
+    cdf_version_header: Literal["alpha", "beta"] | str | None = Field(
+        None, description="The Cognite Data Fusion version header to use for the request."
+    )
+
+
+class CDFTaskParameters(BaseModelResource):
+    cdf_request: CDFRequest
+
+
+class DynamicRef(BaseModelResource):
+    tasks: str = Field(
+        description="A Reference is an expression that allows dynamically injecting input to a task during execution. References can be used to reference the input of the Workflow, the output of a previous task in the Workflow, or the input of a previous task in the Workflow. Note that the injected value must be valid in the context of the property it is injected into. Example Task reference: ${myTaskExternalId.output.someKey} Example Workflow input reference: ${workflow.input.myKey}"
+    )
+
+
+class DynamicTaskParameters(BaseModelResource):
+    dynamic: DynamicRef = Field(description="Reference to another task to use as the definition for this task.")
+
+
+class SubworkflowRef(BaseModelResource):
+    tasks: "WorkflowVersionId | list[Task]" = Field(
+        description="Reference to the subworkflow to execute. This can be either a reference to an existing workflow version or an inline definition of tasks."
+    )
+
+
+class SubworkflowTaskParameters(BaseModelResource):
+    subworkflow: SubworkflowRef = Field(description="Reference to the subworkflow to execute.")
+
+
+class SimulatorInputUnit(BaseModelResource):
+    name: str = Field(description="Name of the unit.")
+
+
+class SimulatorInput(BaseModelResource):
+    reference_id: str = Field(description="Reference id of the value to override.")
+    value: str | int | float | list[str] | list[int] | list[float] = Field(
+        description="Override the value used for a simulation run."
+    )
+    unit: SimulatorInputUnit | None = Field(None, description="Override the unit of the value")
+
+
+class SimulationRef(BaseModelResource):
+    routine_external_id: str = Field(description="The external id of the routine to be executed.")
+    run_time: int | None = Field(
+        None,
+        description="Run time in milliseconds. Reference timestamp used for data pre-processing and data sampling.",
+    )
+    inputs: list[SimulatorInput] | None = Field(
+        None, description="List of inputs to be provided to the simulation.", max_length=200
+    )
+
+
+class SimulationTaskParameters(BaseModelResource):
+    simulation: SimulationRef = Field(description="Reference to the simulation to execute.")
+
+
 class TaskId(BaseModelResource):
     external_id: str = Field(
         max_length=255, description="The external ID provided by the client. Must be unique for the resource type."
     )
 
 
-class TaskDefinition(BaseModelResource):
+class TaskDefinition(BaseModelResource, ABC):
     external_id: str = Field(
         max_length=255,
         description="Identifier for the task. Must be unique within the version. No trailing or leading whitespace and no null characters allowed.",
     )
-    type: Literal["function", "transformation", "cdf", "dynamic", "subworkflow", "simulation"]
+    type: str
     name: str | None = Field(
         default=None,
         max_length=255,
@@ -28,7 +152,6 @@ class TaskDefinition(BaseModelResource):
         max_length=500,
         description="Description of the intention of the task",
     )
-    parameters: JsonValue = Field()
     retries: int = Field(
         3,
         ge=0,
@@ -52,13 +175,49 @@ class TaskDefinition(BaseModelResource):
     )
 
 
+class FunctionTask(TaskDefinition):
+    type: Literal["function"] = "function"
+    parameters: FunctionTaskParameters
+
+
+class TransformationTask(TaskDefinition):
+    type: Literal["transformation"] = "transformation"
+    parameters: TransformationTaskParameters
+
+
+class CDFTask(TaskDefinition):
+    type: Literal["cdfRequest"] = "cdfRequest"
+    parameters: CDFTaskParameters
+
+
+class DynamicTask(TaskDefinition):
+    type: Literal["dynamic"] = "dynamic"
+    parameters: DynamicTaskParameters
+
+
+class SubworkflowTask(TaskDefinition):
+    type: Literal["subworkflow"] = "subworkflow"
+    parameters: SubworkflowTaskParameters
+
+
+class SimulationTask(TaskDefinition):
+    type: Literal["simulation"] = "simulation"
+    parameters: SimulationTaskParameters
+
+
+Task = Annotated[
+    FunctionTask | TransformationTask | CDFTask | DynamicTask | SubworkflowTask | SimulationTask,
+    Field(discriminator="type"),
+]
+
+
 class WorkflowDefinition(BaseModelResource):
     description: str | None = Field(
         default=None,
         max_length=500,
         description="The description of the workflow version.",
     )
-    tasks: list[TaskDefinition]
+    tasks: list[Task]
 
 
 class WorkflowVersionYAML(ToolkitResource):

cognite_toolkit/_cdf_tk/storageio/__init__.py

@@ -1,13 +1,10 @@
-from pathlib import Path
-
 from cognite_toolkit._cdf_tk.utils._auxiliary import get_concrete_subclasses
-from cognite_toolkit._cdf_tk.utils.fileio import COMPRESSION_BY_SUFFIX
 
-from ._annotations import FileAnnotationIO
+from ._annotations import AnnotationIO
 from ._applications import CanvasIO, ChartIO
 from ._asset_centric import (
+    AssetCentricIO,
     AssetIO,
-    BaseAssetCentricIO,
     EventIO,
     FileMetadataIO,
     HierarchyIO,
@@ -25,6 +22,7 @@ from ._base import (
 )
 from ._data_classes import InstanceIdCSVList, InstanceIdRow, ModelList
 from ._datapoints import DatapointsIO
+from ._file_content import FileContentIO
 from ._instances import InstanceIO
 from ._raw import RawIO
 from .selectors._base import DataSelector
@@ -34,33 +32,24 @@ STORAGE_IO_CLASSES = get_concrete_subclasses(StorageIO)  # type: ignore[type-abs
 UPLOAD_IO_CLASSES = get_concrete_subclasses(UploadableStorageIO)  # type: ignore[type-abstract]
 
 
-def get_upload_io(selector_cls: type[DataSelector], kind: str | Path) -> type[UploadableStorageIO]:
+def get_upload_io(selector: DataSelector) -> type[UploadableStorageIO]:
     """Get the appropriate UploadableStorageIO class based on the type of the provided selector."""
     for cls in UPLOAD_IO_CLASSES:
-        if issubclass(selector_cls, cls.BASE_SELECTOR) and are_same_kind(cls.KIND, kind):
+        if isinstance(selector, cls.BASE_SELECTOR) and selector.kind == cls.KIND:
             return cls
-    raise ValueError(f"No UploadableStorageIO found for selector of type {selector_cls.__name__}")
-
-
-def are_same_kind(kind: str, kind_or_path: str | Path, /) -> bool:
-    """Check if two kinds are the same, ignoring case and compression suffixes."""
-    if not isinstance(kind_or_path, Path):
-        return kind.casefold() == kind_or_path.casefold()
-    stem = kind_or_path.stem
-    if kind_or_path.suffix in COMPRESSION_BY_SUFFIX:
-        stem = Path(stem).stem
-    return stem.lower().endswith(kind.casefold())
+    raise ValueError(f"No UploadableStorageIO found for selector of type {type(selector).__name__}")
 
 
 __all__ = [
+    "AnnotationIO",
+    "AssetCentricIO",
     "AssetIO",
-    "BaseAssetCentricIO",
     "CanvasIO",
     "ChartIO",
     "ConfigurableStorageIO",
     "DatapointsIO",
     "EventIO",
-    "FileAnnotationIO",
+    "FileContentIO",
     "FileMetadataIO",
     "HierarchyIO",
     "InstanceIO",
@@ -76,6 +65,5 @@ __all__ = [
     "TimeSeriesIO",
     "UploadItem",
     "UploadableStorageIO",
-    "are_same_kind",
     "get_upload_io",
 ]

cognite_toolkit/_cdf_tk/storageio/_annotations.py

@@ -11,7 +11,7 @@ from ._base import Page, StorageIO
 from .selectors import AssetCentricSelector
 
 
-class FileAnnotationIO(StorageIO[AssetCentricSelector, Annotation]):
+class AnnotationIO(StorageIO[AssetCentricSelector, Annotation]):
     SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
     SUPPORTED_COMPRESSIONS = frozenset({".gz"})
     CHUNK_SIZE = 1000
@@ -23,24 +23,27 @@ class FileAnnotationIO(StorageIO[AssetCentricSelector, Annotation]):
         project = item._cognite_client.config.project
         return f"INTERNAL_ID_project_{project}_{item.id!s}"
 
-    def stream_data(self, selector: AssetCentricSelector, limit: int | None = None) -> Iterable[Page]:
+    def stream_data(self, selector: AssetCentricSelector, limit: int | None = None) -> Iterable[Page[Annotation]]:
         total = 0
         for file_chunk in FileMetadataIO(self.client).stream_data(selector, None):
-            # Todo Support pagination. This is missing in the SDK.
-            results = self.client.annotations.list(
-                filter=AnnotationFilter(
-                    annotated_resource_type="file",
-                    annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+            for annotation_type in ["diagrams.AssetLink", "diagrams.FileLink"]:
+                # Todo Support pagination. This is missing in the SDK.
+                results = self.client.annotations.list(
+                    filter=AnnotationFilter(
+                        annotated_resource_type="file",
+                        annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+                        annotation_type=annotation_type,
+                    ),
+                    limit=-1,
                 )
-            )
-            if limit is not None and total + len(results) > limit:
-                results = results[: limit - total]
-
-            for chunk in chunker_sequence(results, self.CHUNK_SIZE):
-                yield Page(worker_id="main", items=chunk)
-                total += len(chunk)
-            if limit is not None and total >= limit:
-                break
+                if limit is not None and total + len(results) > limit:
+                    results = results[: limit - total]
+
+                for chunk in chunker_sequence(results, self.CHUNK_SIZE):
+                    yield Page(worker_id="main", items=chunk)
+                    total += len(chunk)
+                if limit is not None and total >= limit:
+                    break
 
     def count(self, selector: AssetCentricSelector) -> int | None:
         """There is no efficient way to count annotations in CDF."""