cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/loaders/__init__.py

@@ -1,198 +0,0 @@
-# Copyright 2023 Cognite AS
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import itertools
-from typing import Literal, TypeAlias
-
-from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags
-
-from ._base_loaders import DataLoader, Loader, ResourceContainerLoader, ResourceLoader
-from ._data_loaders import DatapointsLoader, FileLoader, RawFileLoader
-from ._resource_loaders import (
-    AgentLoader,
-    AssetLoader,
-    CogniteFileLoader,
-    ContainerLoader,
-    DataModelLoader,
-    DatapointSubscriptionLoader,
-    DataSetsLoader,
-    EdgeLoader,
-    EventLoader,
-    ExtractionPipelineConfigLoader,
-    ExtractionPipelineLoader,
-    FileMetadataLoader,
-    FunctionLoader,
-    FunctionScheduleLoader,
-    GraphQLLoader,
-    GroupAllScopedLoader,
-    GroupLoader,
-    GroupResourceScopedLoader,
-    HostedExtractorDestinationLoader,
-    HostedExtractorJobLoader,
-    HostedExtractorMappingLoader,
-    HostedExtractorSourceLoader,
-    InfieldV1Loader,
-    LabelLoader,
-    LocationFilterLoader,
-    NodeLoader,
-    RawDatabaseLoader,
-    RawTableLoader,
-    RelationshipLoader,
-    RobotCapabilityLoader,
-    RoboticFrameLoader,
-    RoboticLocationLoader,
-    RoboticMapLoader,
-    RoboticsDataPostProcessingLoader,
-    SecurityCategoryLoader,
-    SequenceLoader,
-    SequenceRowLoader,
-    SpaceLoader,
-    StreamlitLoader,
-    ThreeDModelLoader,
-    TimeSeriesLoader,
-    TransformationLoader,
-    TransformationNotificationLoader,
-    TransformationScheduleLoader,
-    ViewLoader,
-    WorkflowLoader,
-    WorkflowTriggerLoader,
-    WorkflowVersionLoader,
-)
-from ._worker import ResourceWorker
-
-_EXCLUDED_LOADERS: set[type[ResourceLoader]] = set()
-if not FeatureFlag.is_enabled(Flags.GRAPHQL):
-    _EXCLUDED_LOADERS.add(GraphQLLoader)
-if not FeatureFlag.is_enabled(Flags.AGENTS):
-    _EXCLUDED_LOADERS.add(AgentLoader)
-if not FeatureFlag.is_enabled(Flags.INFIELD):
-    _EXCLUDED_LOADERS.add(InfieldV1Loader)
-
-LOADER_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
-for _loader in itertools.chain(
-    ResourceLoader.__subclasses__(),
-    ResourceContainerLoader.__subclasses__(),
-    DataLoader.__subclasses__(),
-    GroupLoader.__subclasses__(),
-):
-    if _loader in [ResourceLoader, ResourceContainerLoader, DataLoader, GroupLoader] or _loader in _EXCLUDED_LOADERS:
-        # Skipping base classes
-        continue
-    if _loader.folder_name not in LOADER_BY_FOLDER_NAME:  # type: ignore[attr-defined]
-        LOADER_BY_FOLDER_NAME[_loader.folder_name] = []  # type: ignore[attr-defined]
-    # MyPy bug: https://github.com/python/mypy/issues/4717
-    LOADER_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[type-abstract, attr-defined, arg-type]
-del _loader  # cleanup module namespace
-
-LOADER_LIST = list(itertools.chain(*LOADER_BY_FOLDER_NAME.values()))
-RESOURCE_LOADER_LIST = [loader for loader in LOADER_LIST if issubclass(loader, ResourceLoader)]
-RESOURCE_CONTAINER_LOADER_LIST = [loader for loader in LOADER_LIST if issubclass(loader, ResourceContainerLoader)]
-RESOURCE_DATA_LOADER_LIST = [loader for loader in LOADER_LIST if issubclass(loader, DataLoader)]
-KINDS_BY_FOLDER_NAME: dict[str, set[str]] = {}
-for loader in LOADER_LIST:
-    if loader.folder_name not in KINDS_BY_FOLDER_NAME:
-        KINDS_BY_FOLDER_NAME[loader.folder_name] = set()
-    KINDS_BY_FOLDER_NAME[loader.folder_name].add(loader.kind)
-del loader  # cleanup module namespace
-
-ResourceTypes: TypeAlias = Literal[  # type: ignore[no-redef, misc]
-    "3dmodels",
-    "agents",
-    "auth",
-    "cdf_applications",
-    "classic",
-    "data_models",
-    "data_sets",
-    "hosted_extractors",
-    "locations",
-    "transformations",
-    "files",
-    "timeseries",
-    "extraction_pipelines",
-    "functions",
-    "raw",
-    "robotics",
-    "streamlit",
-    "workflows",
-]
-
-
-def get_loader(resource_dir: str, kind: str) -> type[Loader]:
-    for loader in LOADER_BY_FOLDER_NAME[resource_dir]:
-        if loader.kind == kind:
-            return loader
-    raise ValueError(f"Loader not found for {resource_dir} and {kind}")
-
-
-__all__ = [
-    "KINDS_BY_FOLDER_NAME",
-    "LOADER_BY_FOLDER_NAME",
-    "LOADER_LIST",
-    "RESOURCE_CONTAINER_LOADER_LIST",
-    "RESOURCE_DATA_LOADER_LIST",
-    "RESOURCE_LOADER_LIST",
-    "AssetLoader",
-    "CogniteFileLoader",
-    "ContainerLoader",
-    "DataLoader",
-    "DataModelLoader",
-    "DataSetsLoader",
-    "DatapointSubscriptionLoader",
-    "DatapointsLoader",
-    "EdgeLoader",
-    "EventLoader",
-    "ExtractionPipelineConfigLoader",
-    "ExtractionPipelineLoader",
-    "FileLoader",
-    "FileMetadataLoader",
-    "FunctionLoader",
-    "FunctionScheduleLoader",
-    "GroupAllScopedLoader",
-    "GroupLoader",
-    "GroupResourceScopedLoader",
-    "HostedExtractorDestinationLoader",
-    "HostedExtractorJobLoader",
-    "HostedExtractorMappingLoader",
-    "HostedExtractorSourceLoader",
-    "LabelLoader",
-    "LocationFilterLoader",
-    "NodeLoader",
-    "RawDatabaseLoader",
-    "RawFileLoader",
-    "RawTableLoader",
-    "RelationshipLoader",
-    "ResourceContainerLoader",
-    "ResourceLoader",
-    "ResourceTypes",
-    "ResourceWorker",
-    "RobotCapabilityLoader",
-    "RoboticFrameLoader",
-    "RoboticLocationLoader",
-    "RoboticMapLoader",
-    "RoboticsDataPostProcessingLoader",
-    "SecurityCategoryLoader",
-    "SequenceLoader",
-    "SequenceRowLoader",
-    "SpaceLoader",
-    "StreamlitLoader",
-    "ThreeDModelLoader",
-    "TimeSeriesLoader",
-    "TransformationLoader",
-    "TransformationNotificationLoader",
-    "TransformationScheduleLoader",
-    "ViewLoader",
-    "WorkflowLoader",
-    "WorkflowTriggerLoader",
-    "WorkflowVersionLoader",
-    "get_loader",
-]

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/__init__.py

@@ -1,90 +0,0 @@
-from .agent_loaders import AgentLoader
-from .auth_loaders import GroupAllScopedLoader, GroupLoader, SecurityCategoryLoader
-from .classic_loaders import AssetLoader, EventLoader, SequenceLoader, SequenceRowLoader
-from .data_organization_loaders import DataSetsLoader, LabelLoader
-from .datamodel_loaders import (
-    ContainerLoader,
-    DataModelLoader,
-    EdgeLoader,
-    GraphQLLoader,
-    NodeLoader,
-    SpaceLoader,
-    ViewLoader,
-)
-from .extraction_pipeline_loaders import ExtractionPipelineConfigLoader, ExtractionPipelineLoader
-from .fieldops_loaders import InfieldV1Loader
-from .file_loader import CogniteFileLoader, FileMetadataLoader
-from .function_loaders import FunctionLoader, FunctionScheduleLoader
-from .group_scoped_loader import GroupResourceScopedLoader
-from .hosted_extractors import (
-    HostedExtractorDestinationLoader,
-    HostedExtractorJobLoader,
-    HostedExtractorMappingLoader,
-    HostedExtractorSourceLoader,
-)
-from .industrial_tool_loaders import StreamlitLoader
-from .location_loaders import LocationFilterLoader
-from .raw_loaders import RawDatabaseLoader, RawTableLoader
-from .relationship_loader import RelationshipLoader
-from .robotics_loaders import (
-    RobotCapabilityLoader,
-    RoboticFrameLoader,
-    RoboticLocationLoader,
-    RoboticMapLoader,
-    RoboticsDataPostProcessingLoader,
-)
-from .three_d_model_loaders import ThreeDModelLoader
-from .timeseries_loaders import DatapointSubscriptionLoader, TimeSeriesLoader
-from .transformation_loaders import TransformationLoader, TransformationNotificationLoader, TransformationScheduleLoader
-from .workflow_loaders import WorkflowLoader, WorkflowTriggerLoader, WorkflowVersionLoader
-
-__all__ = [
-    "AgentLoader",
-    "AssetLoader",
-    "CogniteFileLoader",
-    "ContainerLoader",
-    "DataModelLoader",
-    "DataSetsLoader",
-    "DatapointSubscriptionLoader",
-    "EdgeLoader",
-    "EventLoader",
-    "ExtractionPipelineConfigLoader",
-    "ExtractionPipelineLoader",
-    "FileMetadataLoader",
-    "FunctionLoader",
-    "FunctionScheduleLoader",
-    "GraphQLLoader",
-    "GroupAllScopedLoader",
-    "GroupLoader",
-    "GroupResourceScopedLoader",
-    "HostedExtractorDestinationLoader",
-    "HostedExtractorJobLoader",
-    "HostedExtractorMappingLoader",
-    "HostedExtractorSourceLoader",
-    "InfieldV1Loader",
-    "LabelLoader",
-    "LocationFilterLoader",
-    "NodeLoader",
-    "RawDatabaseLoader",
-    "RawTableLoader",
-    "RelationshipLoader",
-    "RobotCapabilityLoader",
-    "RoboticFrameLoader",
-    "RoboticLocationLoader",
-    "RoboticMapLoader",
-    "RoboticsDataPostProcessingLoader",
-    "SecurityCategoryLoader",
-    "SequenceLoader",
-    "SequenceRowLoader",
-    "SpaceLoader",
-    "StreamlitLoader",
-    "ThreeDModelLoader",
-    "TimeSeriesLoader",
-    "TransformationLoader",
-    "TransformationNotificationLoader",
-    "TransformationScheduleLoader",
-    "ViewLoader",
-    "WorkflowLoader",
-    "WorkflowTriggerLoader",
-    "WorkflowVersionLoader",
-]

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/fieldops_loaders.py

@@ -1,260 +0,0 @@
-import collections.abc
-from collections.abc import Hashable, Iterable
-from functools import lru_cache
-from pathlib import Path
-from typing import Any, final
-
-from cognite.client.data_classes.capabilities import Capability, DataModelInstancesAcl
-from cognite.client.data_classes.data_modeling import NodeApplyResultList, NodeId
-from cognite.client.exceptions import CogniteAPIError
-from cognite.client.utils.useful_types import SequenceNotStr
-
-from cognite_toolkit._cdf_tk._parameters import ParameterSpec, ParameterSpecSet
-from cognite_toolkit._cdf_tk.client.data_classes.apm_config_v1 import (
-    APMConfig,
-    APMConfigList,
-    APMConfigWrite,
-    APMConfigWriteList,
-)
-from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
-from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceLoader
-from cognite_toolkit._cdf_tk.utils import quote_int_value_by_key_in_yaml, safe_read
-from cognite_toolkit._cdf_tk.utils.cdf import iterate_instances
-from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable, hash_dict
-
-from .auth_loaders import GroupAllScopedLoader
-from .classic_loaders import AssetLoader
-from .data_organization_loaders import DataSetsLoader
-from .datamodel_loaders import SpaceLoader
-from .group_scoped_loader import GroupResourceScopedLoader
-
-
-@final
-class InfieldV1Loader(ResourceLoader[str, APMConfigWrite, APMConfig, APMConfigWriteList, APMConfigList]):
-    folder_name = "cdf_applications"
-    filename_pattern = r"^.*\.InfieldV1$"  # Matches all yaml files whose stem ends with '.InfieldV1'.
-    filetypes = frozenset({"yaml", "yml"})
-    resource_cls = APMConfig
-    resource_write_cls = APMConfigWrite
-    list_cls = APMConfigList
-    list_write_cls = APMConfigWriteList
-    kind = "InfieldV1"
-    dependencies = frozenset(
-        {DataSetsLoader, AssetLoader, SpaceLoader, GroupAllScopedLoader, GroupResourceScopedLoader}
-    )
-    _doc_url = "Instances/operation/applyNodeAndEdges"
-    _root_location_filters: tuple[str, ...] = ("general", "assets", "files", "timeseries")
-    _group_keys: tuple[str, ...] = ("templateAdmins", "checklistAdmins")
-
-    @property
-    def display_name(self) -> str:
-        return "infield configs"
-
-    @classmethod
-    def get_id(cls, item: APMConfig | APMConfigWrite | dict) -> str:
-        if isinstance(item, dict):
-            return item["externalId"]
-        if not item.external_id:
-            raise KeyError("APMConfig must have external_id")
-        return item.external_id
-
-    @classmethod
-    def dump_id(cls, id: str) -> dict[str, Any]:
-        return {"externalId": id}
-
-    @classmethod
-    def get_required_capability(
-        cls, items: collections.abc.Sequence[APMConfigWrite] | None, read_only: bool
-    ) -> Capability | list[Capability]:
-        if not items and items is not None:
-            return []
-
-        actions = (
-            [DataModelInstancesAcl.Action.Read]
-            if read_only
-            else [DataModelInstancesAcl.Action.Read, DataModelInstancesAcl.Action.Write]
-        )
-
-        return DataModelInstancesAcl(actions, DataModelInstancesAcl.Scope.SpaceID([APMConfig.space]))
-
-    def create(self, items: APMConfigWriteList) -> NodeApplyResultList:
-        result = self.client.data_modeling.instances.apply(
-            nodes=items.as_nodes(), auto_create_direct_relations=True, replace=False
-        )
-        return result.nodes
-
-    def retrieve(self, ids: SequenceNotStr[str]) -> APMConfigList:
-        result = self.client.data_modeling.instances.retrieve(
-            nodes=self._as_node_ids(ids), sources=APMConfig.view_id
-        ).nodes
-        return APMConfigList.from_nodes(result)
-
-    def update(self, items: APMConfigWriteList) -> NodeApplyResultList:
-        result = self.client.data_modeling.instances.apply(
-            nodes=items.as_nodes(), auto_create_direct_relations=True, replace=True
-        )
-        return result.nodes
-
-    def delete(self, ids: SequenceNotStr[str]) -> int:
-        try:
-            deleted = self.client.data_modeling.instances.delete(nodes=self._as_node_ids(ids))
-        except CogniteAPIError as e:
-            if "not exist" in e.message and "space" in e.message.lower():
-                return 0
-            raise e
-        return len(deleted.nodes)
-
-    @staticmethod
-    def _as_node_ids(ids: SequenceNotStr[str]) -> list[NodeId]:
-        return [NodeId(APMConfig.space, id) for id in ids]
-
-    def _iterate(
-        self,
-        data_set_external_id: str | None = None,
-        space: str | None = None,
-        parent_ids: list[Hashable] | None = None,
-    ) -> Iterable[APMConfig]:
-        for node in iterate_instances(
-            self.client, space=space, instance_type="node", source=APMConfig.view_id, console=self.console
-        ):
-            yield APMConfig.from_node(node)
-
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # Added by toolkit
-        spec.add(
-            ParameterSpec(
-                ("featureConfiguration", "rootLocationConfiguration", "dataSetExternalId"),
-                frozenset({"str"}),
-                is_required=False,
-                _is_nullable=False,
-            )
-        )
-        spec.discard(
-            ParameterSpec(
-                (
-                    "featureConfiguration",
-                    "rootLocationConfiguration",
-                    "dataSetId",
-                ),
-                frozenset({"int"}),
-                is_required=False,
-                _is_nullable=False,
-            )
-        )
-        return spec
-
-    @classmethod
-    def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
-        if isinstance(app_data_space_id := item.get("appDataSpaceId"), str):
-            yield SpaceLoader, app_data_space_id
-        if isinstance(customer_data_space_id := item.get("customerDataSpaceId"), str):
-            yield SpaceLoader, customer_data_space_id
-        for config in cls._get_root_location_configurations(item) or []:
-            if isinstance(asset_external_id := config.get("assetExternalId"), str):
-                yield AssetLoader, asset_external_id
-            if isinstance(data_set_external_id := config.get("dataSetExternalId"), str):
-                yield DataSetsLoader, data_set_external_id
-            if isinstance(app_data_instance_space := config.get("appDataInstanceSpace"), str):
-                yield SpaceLoader, app_data_instance_space
-            if isinstance(source_data_instance_space := config.get("sourceDataInstanceSpace"), str):
-                yield SpaceLoader, source_data_instance_space
-            for key in cls._group_keys:
-                for group in config.get(key, []):
-                    if isinstance(group, str):
-                        yield GroupResourceScopedLoader, group
-            data_filters = config.get("dataFilters")
-            if not isinstance(data_filters, dict):
-                continue
-            for key in cls._root_location_filters:
-                filter_ = data_filters.get(key)
-                if not isinstance(filter_, dict):
-                    continue
-                for data_set_external_id in filter_.get("dataSetExternalIds", []):
-                    if isinstance(data_set_external_id, str):
-                        yield DataSetsLoader, data_set_external_id
-                for asset_external_id in filter_.get("assetSubtreeExternalIds", []):
-                    if isinstance(asset_external_id, str):
-                        yield AssetLoader, asset_external_id
-                if app_data_instance_space := filter_.get("appDataInstanceSpace"):
-                    if isinstance(app_data_instance_space, str):
-                        yield SpaceLoader, app_data_instance_space
-
-    def safe_read(self, filepath: Path | str) -> str:
-        # The customerDataSpaceVersion is a string, but the user often writes it as an int.
-        # YAML will then parse it as an int, for example, `3_0_2` will be parsed as `302`.
-        # This is technically a user mistake, as you should quote the version in the YAML file.
-        # However, we do not want to put this burden on the user (knowing the intricate workings of YAML),
-        # so we fix it here.
-        return quote_int_value_by_key_in_yaml(
-            safe_read(filepath, encoding=BUILD_FOLDER_ENCODING), key="customerDataSpaceVersion"
-        )
-
-    def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> APMConfigWrite:
-        root_location_configurations = self._get_root_location_configurations(resource)
-        for config in root_location_configurations or []:
-            if not isinstance(config, dict):
-                continue
-            if ds_external_id := config.pop("dataSetExternalId", None):
-                config["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
-            data_filters = config.get("dataFilters")
-            if not isinstance(data_filters, dict):
-                continue
-            for key in self._root_location_filters:
-                filter_ = data_filters.get(key)
-                if not isinstance(filter_, dict):
-                    continue
-                if ds_external_ids := filter_.pop("dataSetExternalIds", None):
-                    filter_["dataSetIds"] = self.client.lookup.data_sets.id(ds_external_ids, is_dry_run)
-        return APMConfigWrite._load(resource)
-
-    def dump_resource(self, resource: APMConfig, local: dict[str, Any] | None = None) -> dict[str, Any]:
-        dumped = resource.as_write().dump()
-        local = local or {}
-        if "existingVersion" not in local:
-            # Existing version is typically not set when creating nodes, but we get it back
-            # when we retrieve the node from the server.
-            dumped.pop("existingVersion", None)
-
-        for config in self._get_root_location_configurations(dumped) or []:
-            if not isinstance(config, dict):
-                continue
-            if data_set_id := config.pop("dataSetId", None):
-                config["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
-            data_filters = config.get("dataFilters")
-            if not isinstance(data_filters, dict):
-                continue
-            for key in self._root_location_filters:
-                filter_ = data_filters.get(key)
-                if not isinstance(filter_, dict):
-                    continue
-                if data_set_ids := filter_.pop("dataSetIds", None):
-                    filter_["dataSetExternalIds"] = self.client.lookup.data_sets.external_id(data_set_ids)
-        return dumped
-
-    def diff_list(
-        self, local: list[Any], cdf: list[Any], json_path: tuple[str | int, ...]
-    ) -> tuple[dict[int, int], list[int]]:
-        if json_path == ("featureConfiguration", "rootLocationConfigurations"):
-            return diff_list_identifiable(local, cdf, get_identifier=hash_dict)
-        if not (len(json_path) >= 3 and json_path[:2] == ("featureConfiguration", "rootLocationConfigurations")):
-            return super().diff_list(local, cdf, json_path)
-
-        if len(json_path) == 4 and json_path[-1] in self._group_keys:
-            return diff_list_hashable(local, cdf)
-        if len(json_path) == 5 and json_path[-1] in ("fullWeightModels", "lightWeightModels"):
-            return diff_list_identifiable(local, cdf, get_identifier=hash_dict)
-        if len(json_path) == 6 and json_path[-2] == "dataFilters" and json_path[-1] in self._root_location_filters:
-            return diff_list_hashable(local, cdf)
-        if len(json_path) == 7 and "observations" in json_path and json_path[-1] in ("type", "priority"):
-            return diff_list_identifiable(local, cdf, get_identifier=hash_dict)
-        return super().diff_list(local, cdf, json_path)
-
-    @staticmethod
-    def _get_root_location_configurations(resource: dict[str, Any]) -> list | None:
-        feature_configuration = resource.get("featureConfiguration")
-        if not isinstance(feature_configuration, dict):
-            return None
-        return feature_configuration.get("rootLocationConfigurations")

cognite_toolkit/_cdf_tk/prototypes/_packages/__init__.py

@@ -1 +0,0 @@
-__all__ = ["empty", "examples", "quickstart", "reference"]

cognite_toolkit/_cdf_tk/prototypes/_packages/reference/manifest.yaml

@@ -1,4 +0,0 @@
-title: 'References: reference yaml files for the supported resources'
-modules:
-  references:
-    title: Reference files