cognite-toolkit 0.7.39__py3-none-any.whl → 0.7.51__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_download_app.py

@@ -7,7 +7,7 @@ import typer
 from questionary import Choice
 from rich import print
 
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawTable
 from cognite_toolkit._cdf_tk.commands import DownloadCommand
 from cognite_toolkit._cdf_tk.constants import DATA_DEFAULT_DIR
 from cognite_toolkit._cdf_tk.feature_flags import Flags

cognite_toolkit/_cdf_tk/apps/_dump_app.py

@@ -6,7 +6,7 @@ from cognite.client.data_classes import WorkflowVersionId
 from cognite.client.data_classes.data_modeling import DataModelId, ViewId
 from rich import print
 
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.search_config import ViewId as SearchConfigViewId
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import ViewId as SearchConfigViewId
 from cognite_toolkit._cdf_tk.commands import DumpResourceCommand
 from cognite_toolkit._cdf_tk.commands.dump_resource import (
     AgentFinder,

cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -868,13 +868,13 @@ class MigrateApp(typer.Typer):
                 "performed to select the Canvas to migrate."
             ),
         ] = None,
-        skip_on_missing_ref: Annotated[
+        allow_missing_ref: Annotated[
             bool,
             typer.Option(
-                "--skip-on-missing-ref",
-                "-s",
-                help="If set, the migration will skip Canvases that reference resources that have not been migrated to data modeling. "
-                "If not set, the migration will continue but the result will exclude the missing references.",
+                "--allow-missing-ref",
+                "-a",
+                help="If set, the command will migrate Canvases that reference resources that have not been migrated to data modeling. "
+                "If not set, the migration will fail if any referenced resource are missing.",
             ),
         ] = False,
         log_dir: Annotated[
@@ -926,7 +926,7 @@ class MigrateApp(typer.Typer):
             lambda: cmd.migrate(
                 selected=selector,
                 data=CanvasIO(client, exclude_existing_version=True),
-                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=skip_on_missing_ref),
+                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=not allow_missing_ref),
                 log_dir=log_dir,
                 dry_run=dry_run,
                 verbose=verbose,

cognite_toolkit/_cdf_tk/builders/_function.py

@@ -1,5 +1,8 @@
 import shutil
+import time
 from collections.abc import Callable, Iterable, Sequence
+from pathlib import Path
+from typing import Any
 
 from cognite_toolkit._cdf_tk.builders import Builder
 from cognite_toolkit._cdf_tk.cruds import FunctionCRUD
@@ -10,21 +13,74 @@ from cognite_toolkit._cdf_tk.data_classes import (
     ModuleLocation,
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import (
     FileReadWarning,
+    FunctionRequirementsValidationWarning,
     HighSeverityWarning,
     LowSeverityWarning,
     MediumSeverityWarning,
     ToolkitWarning,
     WarningList,
 )
+from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip
+
+# Maximum number of error lines to display in warnings
+_MAX_ERROR_LINES = 3
 
 
 class FunctionBuilder(Builder):
     _resource_folder = FunctionCRUD.folder_name
 
+    def __init__(self, build_dir: Path, warn: Callable[[ToolkitWarning], None]) -> None:
+        super().__init__(build_dir, warn=warn)
+        # Metrics for telemetry
+        self.validation_count = 0
+        self.validation_failures = 0
+        self.validation_credential_errors = 0
+        self.validation_time_ms = 0
+
+    def _validate_function_requirements(
+        self,
+        requirements_txt: Path,
+        raw_function: dict[str, Any],
+        filepath: Path,
+        external_id: str,
+    ) -> FunctionRequirementsValidationWarning | None:
+        """Validate function requirements.txt using pip dry-run."""
+        start_time = time.time()
+        validation_result = validate_requirements_with_pip(
+            requirements_txt_path=requirements_txt,
+            index_url=raw_function.get("indexUrl"),
+            extra_index_urls=raw_function.get("extraIndexUrls"),
+        )
+        elapsed_ms = int((time.time() - start_time) * 1000)
+        self.validation_count += 1
+        self.validation_time_ms += elapsed_ms
+
+        if validation_result.success:
+            return None
+
+        self.validation_failures += 1
+        if validation_result.is_credential_error:
+            self.validation_credential_errors += 1
+
+        error_detail = validation_result.error_message or "Unknown error"
+        relevant_lines = [line for line in error_detail.strip().split("\n") if line.strip()][-_MAX_ERROR_LINES:]
+        error_detail = "\n      ".join(relevant_lines)
+
+        return FunctionRequirementsValidationWarning(
+            filepath=filepath,
+            function_external_id=external_id,
+            error_details=error_detail,
+            is_credential_error=validation_result.is_credential_error,
+        )
+
     def build(
-        self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None
+        self,
+        source_files: list[BuildSourceFile],
+        module: ModuleLocation,
+        console: Callable[[str], None] | None = None,
     ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]:
         for source_file in source_files:
             if source_file.loaded is None:
@@ -55,7 +111,9 @@ class FunctionBuilder(Builder):
             )
 
     def validate_directory(
-        self, built_resources: BuiltResourceList, module: ModuleLocation
+        self,
+        built_resources: BuiltResourceList,
+        module: ModuleLocation,
     ) -> WarningList[ToolkitWarning]:
         warnings = WarningList[ToolkitWarning]()
         has_config_files = any(resource.kind == FunctionCRUD.kind for resource in built_resources)
@@ -74,7 +132,7 @@ class FunctionBuilder(Builder):
                     f"was not found in {required_location.as_posix()!r}. "
                     f"The file {yaml_source_path.as_posix()!r} is currently "
                     f"considered part of the Function's artifacts and "
-                    f"will not be processed by the Toolkit."
+                    f"will not be processed by the Toolkit.",
                 )
                 warnings.append(warning)
         return warnings
@@ -93,28 +151,42 @@ class FunctionBuilder(Builder):
                 warnings.append(
                     HighSeverityWarning(
                         f"Function in {source_file.source.path.as_posix()!r} has no externalId defined. "
-                        f"This is used to match the function to the function directory."
-                    )
+                        f"This is used to match the function to the function directory.",
+                    ),
                 )
                 continue
             if not function_path:
                 warnings.append(
                     MediumSeverityWarning(
-                        f"Function {external_id} in {source_file.source.path.as_posix()!r} has no function_path defined."
-                    )
+                        f"Function {external_id} in {source_file.source.path.as_posix()!r} has no function_path defined.",
+                    ),
                 )
 
             function_directory = source_file.source.path.with_name(external_id)
 
             if not function_directory.is_dir():
                 raise ToolkitNotADirectoryError(
-                    f"Function directory not found for externalId {external_id} defined in {source_file.source.path.as_posix()!r}."
+                    f"Function directory not found for externalId {external_id} defined in {source_file.source.path.as_posix()!r}.",
+                )
+
+            # Validate requirements.txt if present and feature is enabled
+            if (
+                Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled()
+                and (requirements_txt := function_directory / "requirements.txt").exists()
+            ):
+                warning = self._validate_function_requirements(
+                    requirements_txt,
+                    raw_function,
+                    source_file.source.path,
+                    external_id,
                 )
+                if warning:
+                    warnings.append(warning)
 
             destination = self.build_dir / self.resource_folder / external_id
             if destination.exists():
                 raise ToolkitFileExistsError(
-                    f"Function {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory."
+                    f"Function {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory.",
                 )
             shutil.copytree(function_directory, destination, ignore=shutil.ignore_patterns("__pycache__"))
 

cognite_toolkit/_cdf_tk/builders/_raw.py

@@ -3,7 +3,7 @@ from collections.abc import Callable, Iterable, Sequence
 from typing import Any
 
 from cognite_toolkit._cdf_tk.builders import Builder
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawDatabase
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawDatabase
 from cognite_toolkit._cdf_tk.cruds import RawDatabaseCRUD, RawTableCRUD, ResourceCRUD
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildDestinationFile,

cognite_toolkit/_cdf_tk/client/_resource_base.py

@@ -0,0 +1,187 @@
+"""Base classes for Cognite Client resources and identifiers."""
+
+import sys
+import types
+from abc import ABC, abstractmethod
+from collections import UserList
+from typing import Any, ClassVar, Generic, Literal, TypeVar, Union, get_args, get_origin
+
+from cognite.client import CogniteClient
+from pydantic import BaseModel, ConfigDict
+from pydantic.alias_generators import to_camel
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class BaseModelObject(BaseModel):
+    """Base class for all object. This includes resources and nested objects."""
+
+    # We allow extra fields to support forward compatibility.
+    model_config = ConfigDict(alias_generator=to_camel, extra="allow", populate_by_name=True)
+
+    def dump(self, camel_case: bool = True, exclude_extra: bool = False) -> dict[str, Any]:
+        """Dump the resource to a dictionary.
+
+        Args:
+            camel_case (bool): Whether to use camelCase for the keys. Default is True.
+            exclude_extra (bool): Whether to exclude extra fields not defined in the model. Default is False.
+
+        """
+        if exclude_extra:
+            return self.model_dump(
+                mode="json",
+                by_alias=camel_case,
+                exclude_unset=True,
+                exclude=set(self.__pydantic_extra__) if self.__pydantic_extra__ else None,
+            )
+        return self.model_dump(mode="json", by_alias=camel_case, exclude_unset=True)
+
+    @classmethod
+    def _load(cls, resource: dict[str, Any]) -> Self:
+        """Load method to match CogniteResource signature."""
+        return cls.model_validate(resource, by_alias=True)
+
+
+class RequestItem(BaseModelObject, ABC):
+    """A request item is any object that can be sent to the CDF API as part of a request."""
+
+    def __str__(self) -> str:
+        """All request items must implement a string representation.
+
+        This is used to identify the item in error messages and logs.
+        """
+        raise NotImplementedError()
+
+
+T_RequestItem = TypeVar("T_RequestItem", bound=RequestItem)
+
+
+class Identifier(RequestItem):
+    """Base class for all identifier objects typically
+    {"externalId": "..."}, {"id": ...}, {"space": "...", "externalId: "..."}."""
+
+    model_config = ConfigDict(alias_generator=to_camel, extra="ignore", frozen=True)
+
+    def dump(self, camel_case: bool = True, exclude_extra: bool = False) -> dict[str, Any]:
+        """Dump the resource to a dictionary.
+
+        This is the default serialization method for request resources.
+        """
+        return self.model_dump(mode="json", by_alias=camel_case, exclude_unset=True)
+
+
+T_Identifier = TypeVar("T_Identifier", bound=Identifier)
+
+
+class RequestResource(RequestItem, ABC):
+    @abstractmethod
+    def as_id(self) -> Identifier:
+        raise NotImplementedError()
+
+    def __str__(self) -> str:
+        return str(self.as_id())
+
+
+T_RequestResource = TypeVar("T_RequestResource", bound=RequestResource)
+
+
+class UpdatableRequestResource(RequestResource, ABC):
+    container_fields: ClassVar[frozenset[str]] = frozenset()
+    non_nullable_fields: ClassVar[frozenset[str]] = frozenset()
+
+    def as_update(self, mode: Literal["patch", "replace"]) -> dict[str, Any]:
+        """Convert the request resource to an update item."""
+        update_item = self.as_id().dump(camel_case=True)
+        update: dict[str, Any] = {}
+        field_by_name = {info.alias or field_id: (field_id, info) for field_id, info in type(self).model_fields.items()}
+        # When mode is "patch", we only include fields that are set
+        exclude_unset = mode == "patch"
+        for key, value in self.model_dump(mode="json", by_alias=True, exclude_unset=exclude_unset).items():
+            if key in update_item:
+                # Skip identifier fields
+                continue
+            if key not in field_by_name:
+                # Skip unknown fields
+                continue
+            field_id, info = field_by_name[key]
+            if field_id in self.container_fields:
+                if mode == "patch":
+                    update[key] = {"add": value}
+                elif mode == "replace":
+                    if value is None:
+                        origin = _get_annotation_origin(info.annotation)
+                        if origin is list:
+                            update[key] = {"set": []}
+                        elif origin is dict:
+                            update[key] = {"set": {}}
+                        else:
+                            raise NotImplementedError(
+                                f'Cannot replace container field "{key}" with None when its type is unknown.'
+                            )
+                    else:
+                        update[key] = {"set": value}
+                else:
+                    raise NotImplementedError(f'Update mode "{mode}" is not supported for container fields.')
+            elif value is None:
+                if field_id not in self.non_nullable_fields:
+                    update[key] = {"setNull": True}
+            else:
+                update[key] = {"set": value}
+        update_item["update"] = update
+        return update_item
+
+
+def _get_annotation_origin(field_type: Any) -> Any:
+    origin = get_origin(field_type)
+    args = get_args(field_type)
+
+    # Check for Union type (both typing.Union and | syntax from Python 3.10+)
+    is_union = origin is Union or isinstance(field_type, getattr(types, "UnionType", ()))
+
+    if is_union:
+        # Handle Optional[T] by filtering out NoneType
+        none_types = (type(None), types.NoneType)
+        non_none_args = [arg for arg in args if arg not in none_types]
+        if len(non_none_args) == 1:
+            field_type = non_none_args[0]
+            origin = get_origin(field_type) or field_type
+    return origin
+
+
+class ResponseResource(BaseModelObject, Generic[T_RequestResource], ABC):
+    @abstractmethod
+    def as_request_resource(self) -> T_RequestResource:
+        """Convert the response resource to a request resource."""
+        raise NotImplementedError()
+
+    # Todo remove when CogniteClient data classes are completely removed from the codebase
+    # and we only use the pydantic resource classes instead.from
+    def as_write(self) -> T_RequestResource:
+        """Alias for as_request_resource to match protocol signature."""
+        return self.as_request_resource()
+
+
+T_ResponseResource = TypeVar("T_ResponseResource", bound=ResponseResource)
+
+# Todo: Delete this class and use list[T_Resource] directly
+T_Resource = TypeVar("T_Resource", bound=RequestResource | ResponseResource)
+
+
+class BaseResourceList(UserList[T_Resource]):
+    """Base class for resource lists."""
+
+    _RESOURCE: type[T_Resource]
+
+    def __init__(self, initlist: list[T_Resource] | None = None, **_: Any) -> None:
+        super().__init__(initlist or [])
+
+    def dump(self, camel_case: bool = True) -> list[dict[str, Any]]:
+        return [item.dump(camel_case) for item in self.data]
+
+    @classmethod
+    def load(cls, data: list[dict[str, Any]], cognite_client: "CogniteClient | None" = None) -> Self:
+        items = [cls._RESOURCE.model_validate(item) for item in data]
+        return cls(items)  # type: ignore[arg-type]

cognite_toolkit/_cdf_tk/client/_toolkit_client.py

@@ -11,18 +11,34 @@ from cognite_toolkit._cdf_tk.client.api.legacy.extended_files import ExtendedFil
 from cognite_toolkit._cdf_tk.client.api.legacy.extended_functions import ExtendedFunctionsAPI
 from cognite_toolkit._cdf_tk.client.api.legacy.extended_raw import ExtendedRawAPI
 from cognite_toolkit._cdf_tk.client.api.legacy.extended_timeseries import ExtendedTimeSeriesAPI
-from cognite_toolkit._cdf_tk.client.api.legacy.robotics import RoboticsAPI
-from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient
+from cognite_toolkit._cdf_tk.client.api.legacy.robotics import RoboticsAPI as RoboticsLegacyAPI
+from cognite_toolkit._cdf_tk.client.http_client import HTTPClient
 
+from .api.assets import AssetsAPI
+from .api.datasets import DataSetsAPI
+from .api.events import EventsAPI
+from .api.extraction_pipelines import ExtractionPipelinesAPI
+from .api.filemetadata import FileMetadataAPI
+from .api.hosted_extractors import HostedExtractorsAPI
 from .api.infield import InfieldAPI
+from .api.instances import InstancesAPI
+from .api.labels import LabelsAPI
 from .api.lookup import LookUpGroup
 from .api.migration import MigrationAPI
 from .api.project import ProjectAPI
+from .api.raw import RawAPI
+from .api.robotics import RoboticsAPI
 from .api.search import SearchAPI
+from .api.security_categories import SecurityCategoriesAPI
+from .api.sequences import SequencesAPI
+from .api.simulators import SimulatorsAPI
 from .api.streams import StreamsAPI
 from .api.three_d import ThreeDAPI
+from .api.timeseries import TimeSeriesAPI
 from .api.token import TokenAPI
+from .api.transformations import TransformationsAPI
 from .api.verify import VerifyAPI
+from .api.workflows import WorkflowsAPI
 from .config import ToolkitClientConfig
 
 
@@ -31,7 +47,23 @@ class ToolAPI:
 
     def __init__(self, http_client: HTTPClient, console: Console) -> None:
         self.http_client = http_client
-        self.three_d = ThreeDAPI(http_client, console)
+        self.assets = AssetsAPI(http_client)
+        self.datasets = DataSetsAPI(http_client)
+        self.events = EventsAPI(http_client)
+        self.extraction_pipelines = ExtractionPipelinesAPI(http_client)
+        self.hosted_extractors = HostedExtractorsAPI(http_client)
+        self.instances = InstancesAPI(http_client)
+        self.labels = LabelsAPI(http_client)
+        self.filemetadata = FileMetadataAPI(http_client)
+        self.raw = RawAPI(http_client)
+        self.robotics = RoboticsAPI(http_client)
+        self.security_categories = SecurityCategoriesAPI(http_client)
+        self.sequences = SequencesAPI(http_client)
+        self.simulators = SimulatorsAPI(http_client)
+        self.three_d = ThreeDAPI(http_client)
+        self.timeseries = TimeSeriesAPI(http_client)
+        self.transformations = TransformationsAPI(http_client)
+        self.workflows = WorkflowsAPI(http_client)
 
 
 class ToolkitClient(CogniteClient):
@@ -48,7 +80,7 @@ class ToolkitClient(CogniteClient):
         self.console = console or Console()
         self.tool = ToolAPI(http_client, self.console)
         self.search = SearchAPI(self._config, self._API_VERSION, self)
-        self.robotics = RoboticsAPI(self._config, self._API_VERSION, self)
+        self.robotics = RoboticsLegacyAPI(self._config, self._API_VERSION, self)
         self.dml = DMLAPI(self._config, self._API_VERSION, self)
         self.verify = VerifyAPI(self._config, self._API_VERSION, self)
         self.lookup = LookUpGroup(self._config, self._API_VERSION, self, self.console)
@@ -66,7 +98,7 @@ class ToolkitClient(CogniteClient):
         self.charts = ChartsAPI(self._config, self._API_VERSION, self)
         self.project = ProjectAPI(config=toolkit_config, cognite_client=self)
         self.infield = InfieldAPI(http_client, self.console)
-        self.streams = StreamsAPI(http_client, self.console)
+        self.streams = StreamsAPI(http_client)
 
     @property
     def config(self) -> ToolkitClientConfig:

cognite_toolkit/_cdf_tk/client/api/agents.py

@@ -0,0 +1,107 @@
+"""Agents API for managing CDF AI agents.
+
+Based on the API specification at:
+https://api-docs.cognite.com/20230101-alpha/tag/Agents/operation/main_ai_agents_post
+
+Note: This is an alpha API and may change in future releases.
+"""
+
+from collections.abc import Iterable, Sequence
+
+from cognite_toolkit._cdf_tk.client.cdf_client import CDFResourceAPI, Endpoint, PagedResponse
+from cognite_toolkit._cdf_tk.client.http_client import HTTPClient, ItemsSuccessResponse2, SuccessResponse2
+from cognite_toolkit._cdf_tk.client.resource_classes.agent import AgentRequest, AgentResponse
+from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId
+
+
+class AgentsAPI(CDFResourceAPI[ExternalId, AgentRequest, AgentResponse]):
+    """API for managing CDF AI agents.
+
+    Note: This is an alpha API and may change in future releases.
+    """
+
+    def __init__(self, http_client: HTTPClient) -> None:
+        super().__init__(
+            http_client=http_client,
+            method_endpoint_map={
+                "upsert": Endpoint(method="POST", path="/ai/agents", item_limit=1),
+                "retrieve": Endpoint(method="POST", path="/ai/agents/byids", item_limit=100),
+                "delete": Endpoint(method="POST", path="/ai/agents/delete", item_limit=1),
+                "list": Endpoint(method="GET", path="/ai/agents", item_limit=1000),
+            },
+        )
+
+    def _validate_page_response(
+        self, response: SuccessResponse2 | ItemsSuccessResponse2
+    ) -> PagedResponse[AgentResponse]:
+        return PagedResponse[AgentResponse].model_validate_json(response.body)
+
+    def create(self, items: Sequence[AgentRequest]) -> list[AgentResponse]:
+        """Apply (create or update) agents in CDF.
+
+        Args:
+            items: List of AgentRequest objects to apply.
+
+        Returns:
+            List of applied AgentResponse objects.
+        """
+        return self._request_item_response(items, "upsert")
+
+    def update(self, items: Sequence[AgentRequest]) -> list[AgentResponse]:
+        """Update agents in CDF.
+
+        Args:
+            items: List of AgentRequest objects to update.
+        Returns:
+            List of updated AgentResponse objects.
+        """
+        # Implemented as an alias to create (upsert) to have a standardized interface.
+        return self._request_item_response(items, "upsert")
+
+    def retrieve(self, items: Sequence[ExternalId], ignore_unknown_ids: bool = False) -> list[AgentResponse]:
+        """Retrieve agents from CDF by external ID.
+
+        Args:
+            items: List of ExternalId objects to retrieve.
+            ignore_unknown_ids: Whether to ignore unknown IDs.
+
+        Returns:
+            List of retrieved AgentResponse objects.
+        """
+        return self._request_item_response(
+            items, method="retrieve", extra_body={"ignoreUnknownIds": ignore_unknown_ids}
+        )
+
+    def delete(self, items: Sequence[ExternalId], ignore_unknown_ids: bool = False) -> None:
+        """Delete agents from CDF.
+
+        Args:
+            items: List of ExternalId objects to delete.
+            ignore_unknown_ids: Whether to ignore unknown IDs.
+        """
+        self._request_no_response(items, "delete", extra_body={"ignoreUnknownIds": ignore_unknown_ids})
+
+    def iterate(
+        self,
+        limit: int | None = None,
+    ) -> Iterable[list[AgentResponse]]:
+        """Iterate over all agents in CDF.
+
+        Args:
+            limit: Maximum total number of agents to return.
+
+        Returns:
+            Iterable of lists of AgentResponse objects.
+        """
+        return self._iterate(limit=limit)
+
+    def list(self, limit: int | None = None) -> list[AgentResponse]:
+        """List all agents in CDF.
+
+        Args:
+            limit: Maximum total number of agents to return.
+
+        Returns:
+            List of AgentResponse objects.
+        """
+        return self._list(limit=limit)