cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/constants.py

@@ -2,11 +2,12 @@ import os
 import re
 import sys
 from pathlib import Path
+from typing import Literal, TypeAlias
 
-from typing_extensions import Literal, TypeAlias
+from cognite.client.data_classes.data_modeling import ContainerId
 
 try:
-    from pyodide.ffi import IN_BROWSER  # type: ignore [import-not-found]
+    from pyodide.ffi import IN_BROWSER
 except ModuleNotFoundError:
     IN_BROWSER = False
 # This is the default config located locally in each module.
@@ -29,15 +30,16 @@ CONFIG_FILE_SUFFIX = "config.yaml"
 # The global config file
 GLOBAL_CONFIG_FILE = "global.yaml"
 
-BUILTIN_MODULES = "_builtin_modules"
 COGNITE_MODULES = "cognite_modules"
 CUSTOM_MODULES = "custom_modules"
+EXTERNAL_PACKAGE = "external_package"
 MODULES = "modules"
 REPO_FILES_DIR = "_repo_files"
 DOCKER_IMAGE_NAME = "cognite/toolkit"
 BUILD_FOLDER_ENCODING = "utf-8"
+RESOURCES = "_resources"
 
-ROOT_MODULES = [MODULES, CUSTOM_MODULES, COGNITE_MODULES]
+ROOT_MODULES = [MODULES, CUSTOM_MODULES, COGNITE_MODULES, EXTERNAL_PACKAGE]
 MODULE_PATH_SEP = "/"
 
 MIN_TIMESTAMP_MS = -2208988800000  # 1900-01-01 00:00:00.000
@@ -58,8 +60,8 @@ TABLE_FORMATS = frozenset([".csv", ".parquet"])
 ROOT_PATH = Path(__file__).parent.parent
 COGNITE_MODULES_PATH = ROOT_PATH / COGNITE_MODULES
 MODULES_PATH = ROOT_PATH / MODULES
-BUILTIN_MODULES_PATH = ROOT_PATH / BUILTIN_MODULES
-SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.2.0"
+RESOURCES_PATH = ROOT_PATH / RESOURCES
+SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.4.0"
 # This is used in the build directory to keep track of order and flatten the
 # module directory structure with accounting for duplicated names.
 INDEX_PATTERN = re.compile("^[0-9]+\\.")
@@ -77,6 +79,51 @@ HINT_LEAD_TEXT = "[bold blue]HINT[/bold blue] "
 HINT_LEAD_TEXT_LEN = 5
 EnvType: TypeAlias = Literal["dev", "test", "staging", "qa", "prod"]
 USE_SENTRY = "pytest" not in sys.modules and os.environ.get("SENTRY_ENABLED", "true").lower() == "true"
+SPACE_FORMAT_PATTERN = r"^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$"
+DM_EXTERNAL_ID_PATTERN = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
+INSTANCE_EXTERNAL_ID_PATTERN = r"^[^\x00]{1,256}$"
+FORBIDDEN_SPACES = frozenset(["space", "cdf", "dms", "pg3", "shared", "system", "node", "edge"])
+FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS = frozenset(
+    [
+        "Query",
+        "Mutation",
+        "Subscription",
+        "String",
+        "Int32",
+        "Int64",
+        "Int",
+        "Float32",
+        "Float64",
+        "Float",
+        "Timestamp",
+        "JSONObject",
+        "Date",
+        "Numeric",
+        "Boolean",
+        "PageInfo",
+        "File",
+        "Sequence",
+        "TimeSeries",
+    ]
+)
+FORBIDDEN_CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER = frozenset(
+    [
+        "space",
+        "externalId",
+        "createdTime",
+        "lastUpdatedTime",
+        "deletedTime",
+        "edge_id",
+        "node_id",
+        "project_id",
+        "property_group",
+        "seq",
+        "tg_table_name",
+        "extensions",
+    ]
+)
+CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN = r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,253}[a-zA-Z0-9]?$"
+DM_VERSION_PATTERN = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
 
 
 def clean_name(name: str) -> str:
@@ -86,9 +133,58 @@ def clean_name(name: str) -> str:
 
 class URL:
     configure_access = "https://docs.cognite.com/cdf/deploy/cdf_deploy/cdf_deploy_access_management"
-    auth_toolkit = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/guides/configure_deploy_modules#configure-the-cdf-toolkit-authentication"
+    auth_toolkit = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/guides/auth"
     docs = "https://docs.cognite.com/"
-    configs = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/configs"
+    configs = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library"
     plugins = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/guides/plugins/"
     libyaml = "https://pyyaml.org/wiki/PyYAMLDocumentation"
     build_variables = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/api/config_yaml#the-variables-section"
+
+
+# The number of instances that should be left as a margin when Toolkit writes to CDF through the DMS API.
+# This is currently set conservatively to 1 million. The reasoning for this is that there are CDF
+# applications such as Infield and Industrial Canvas that can write to the DMS API, as well as likely third-party
+# applications that can write to the DMS API. If Toolkit fills up the entire capacity, these type of data gathering
+# applications will experience data loss. The limit of 1 million is chosen such that it will trigger alarms in the
+# CDF projects, such that admins can take action to increase or clean up the capacity before it is too late.
+DMS_INSTANCE_LIMIT_MARGIN = 1_000_000
+
+# This is the maximum number of rows that can be iterated over in a single query
+# agreed upon with the transformations team.
+MAX_ROW_ITERATION_RUN_QUERY = 500_000
+MAX_RUN_QUERY_FREQUENCY_MIN = 10
+
+
+COGNITE_MIGRATION_SPACE = "cognite_migration"
+
+COGNITE_TIME_SERIES_CONTAINER = ContainerId("cdf_cdm", "CogniteTimeSeries")
+COGNITE_FILE_CONTAINER = ContainerId("cdf_cdm", "CogniteFile")
+CDF_UNIT_SPACE = "cdf_cdm_units"
+
+
+# Container properties that are read-only in DMS needs to be handled with extra care, as this aspect is not currently exposed by the API.
+READONLY_CONTAINER_PROPERTIES = {
+    ContainerId(space="cdf_cdm", external_id="CogniteAsset"): {
+        "assetHierarchy_path_last_updated_time",
+        "assetHierarchy_path",
+        "assetHierarchy_root",
+    },
+    ContainerId(space="cdf_cdm", external_id="CogniteFile"): {"isUploaded", "uploadedTime"},
+}
+
+# Data Plugin Constants
+DATA_DEFAULT_DIR = "data"
+DATA_RESOURCE_DIR = "resources"
+DATA_MANIFEST_STEM = "Manifest"
+DATA_MANIFEST_SUFFIX = f".{DATA_MANIFEST_STEM}.yaml"
+
+DATA_UPLOAD_URL = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/guides/plugins/data_plugin/index"
+# Migration Constants
+MISSING_INSTANCE_SPACE = "<InstanceSpaceMissing>"
+MISSING_EXTERNAL_ID = "INTERNAL_ID_project_{project}_id_{id}"
+
+# Stream Template names
+STREAM_IMMUTABLE_TEMPLATE_NAME = frozenset(("ImmutableTestStream", "BasicArchive"))
+STREAM_MUTABLE_TEMPLATE_NAME = frozenset(("BasicLiveData",))
+
+StreamTemplateName: TypeAlias = Literal["ImmutableTestStream", "BasicArchive", "BasicLiveData"]

cognite_toolkit/_cdf_tk/cruds/__init__.py

@@ -0,0 +1,224 @@
+# Copyright 2023 Cognite AS
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import itertools
+from typing import Literal, TypeAlias
+
+from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags
+
+from ._base_cruds import DataCRUD, Loader, ResourceContainerCRUD, ResourceCRUD
+from ._data_cruds import DatapointsCRUD, FileCRUD, RawFileCRUD
+from ._resource_cruds import (
+    AgentCRUD,
+    AssetCRUD,
+    CogniteFileCRUD,
+    ContainerCRUD,
+    DataModelCRUD,
+    DatapointSubscriptionCRUD,
+    DataSetsCRUD,
+    EdgeCRUD,
+    EventCRUD,
+    ExtractionPipelineConfigCRUD,
+    ExtractionPipelineCRUD,
+    FileMetadataCRUD,
+    FunctionCRUD,
+    FunctionScheduleCRUD,
+    GraphQLCRUD,
+    GroupAllScopedCRUD,
+    GroupCRUD,
+    GroupResourceScopedCRUD,
+    HostedExtractorDestinationCRUD,
+    HostedExtractorJobCRUD,
+    HostedExtractorMappingCRUD,
+    HostedExtractorSourceCRUD,
+    InFieldCDMLocationConfigCRUD,
+    InFieldLocationConfigCRUD,
+    InfieldV1CRUD,
+    LabelCRUD,
+    LocationFilterCRUD,
+    NodeCRUD,
+    RawDatabaseCRUD,
+    RawTableCRUD,
+    RelationshipCRUD,
+    ResourceViewMappingCRUD,
+    RobotCapabilityCRUD,
+    RoboticFrameCRUD,
+    RoboticLocationCRUD,
+    RoboticMapCRUD,
+    RoboticsDataPostProcessingCRUD,
+    SearchConfigCRUD,
+    SecurityCategoryCRUD,
+    SequenceCRUD,
+    SequenceRowCRUD,
+    SpaceCRUD,
+    StreamCRUD,
+    StreamlitCRUD,
+    ThreeDModelCRUD,
+    TimeSeriesCRUD,
+    TransformationCRUD,
+    TransformationNotificationCRUD,
+    TransformationScheduleCRUD,
+    ViewCRUD,
+    WorkflowCRUD,
+    WorkflowTriggerCRUD,
+    WorkflowVersionCRUD,
+)
+from ._worker import ResourceWorker
+
+_EXCLUDED_CRUDS: set[type[ResourceCRUD]] = set()
+if not FeatureFlag.is_enabled(Flags.GRAPHQL):
+    _EXCLUDED_CRUDS.add(GraphQLCRUD)
+if not FeatureFlag.is_enabled(Flags.INFIELD):
+    _EXCLUDED_CRUDS.add(InfieldV1CRUD)
+    _EXCLUDED_CRUDS.add(InFieldLocationConfigCRUD)
+    _EXCLUDED_CRUDS.add(InFieldCDMLocationConfigCRUD)
+if not FeatureFlag.is_enabled(Flags.MIGRATE):
+    _EXCLUDED_CRUDS.add(ResourceViewMappingCRUD)
+if not FeatureFlag.is_enabled(Flags.STREAMS):
+    _EXCLUDED_CRUDS.add(StreamCRUD)
+
+
+CRUDS_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
+for _loader in itertools.chain(
+    ResourceCRUD.__subclasses__(),
+    ResourceContainerCRUD.__subclasses__(),
+    DataCRUD.__subclasses__(),
+    GroupCRUD.__subclasses__(),
+):
+    if _loader in [ResourceCRUD, ResourceContainerCRUD, DataCRUD, GroupCRUD] or _loader in _EXCLUDED_CRUDS:
+        # Skipping base classes
+        continue
+    if _loader.folder_name not in CRUDS_BY_FOLDER_NAME:  # type: ignore[attr-defined]
+        CRUDS_BY_FOLDER_NAME[_loader.folder_name] = []  # type: ignore[attr-defined]
+    # MyPy bug: https://github.com/python/mypy/issues/4717
+    CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
+del _loader  # cleanup module namespace
+
+# For backwards compatibility
+CRUDS_BY_FOLDER_NAME["data_models"] = CRUDS_BY_FOLDER_NAME["data_modeling"]  # Todo: Remove in v1.0
+RESOURCE_CRUD_BY_FOLDER_NAME = {
+    folder_name: cruds
+    for folder_name, loaders in CRUDS_BY_FOLDER_NAME.items()
+    if (cruds := [crud for crud in loaders if issubclass(crud, ResourceCRUD)])
+}
+
+CRUD_LIST = list(itertools.chain.from_iterable(CRUDS_BY_FOLDER_NAME.values()))
+RESOURCE_CRUD_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceCRUD)]
+RESOURCE_CRUD_CONTAINER_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceContainerCRUD)]
+RESOURCE_DATA_CRUD_LIST = [loader for loader in CRUD_LIST if issubclass(loader, DataCRUD)]
+KINDS_BY_FOLDER_NAME: dict[str, set[str]] = {}
+for crud in CRUD_LIST:
+    if crud.folder_name not in KINDS_BY_FOLDER_NAME:
+        KINDS_BY_FOLDER_NAME[crud.folder_name] = set()
+    KINDS_BY_FOLDER_NAME[crud.folder_name].add(crud.kind)
+del crud  # cleanup module namespace
+
+ResourceTypes: TypeAlias = Literal[
+    "3dmodels",
+    "agents",
+    "auth",
+    "cdf_applications",
+    "classic",
+    "data_modeling",
+    "data_models",  # Todo: Remove in v1.0
+    "data_sets",
+    "hosted_extractors",
+    "locations",
+    "migration",
+    "transformations",
+    "files",
+    "timeseries",
+    "extraction_pipelines",
+    "functions",
+    "raw",
+    "robotics",
+    "streams",
+    "streamlit",
+    "workflows",
+]
+
+
+def get_crud(resource_dir: str, kind: str) -> type[Loader]:
+    for loader in CRUDS_BY_FOLDER_NAME[resource_dir]:
+        if loader.kind == kind:
+            return loader
+    raise ValueError(f"Loader not found for {resource_dir} and {kind}")
+
+
+__all__ = [
+    "CRUDS_BY_FOLDER_NAME",
+    "CRUD_LIST",
+    "KINDS_BY_FOLDER_NAME",
+    "RESOURCE_CRUD_BY_FOLDER_NAME",
+    "RESOURCE_CRUD_CONTAINER_LIST",
+    "RESOURCE_CRUD_LIST",
+    "RESOURCE_DATA_CRUD_LIST",
+    "AgentCRUD",
+    "AssetCRUD",
+    "CogniteFileCRUD",
+    "ContainerCRUD",
+    "DataCRUD",
+    "DataModelCRUD",
+    "DataSetsCRUD",
+    "DatapointSubscriptionCRUD",
+    "DatapointsCRUD",
+    "EdgeCRUD",
+    "EventCRUD",
+    "ExtractionPipelineCRUD",
+    "ExtractionPipelineConfigCRUD",
+    "FileCRUD",
+    "FileMetadataCRUD",
+    "FunctionCRUD",
+    "FunctionScheduleCRUD",
+    "GroupAllScopedCRUD",
+    "GroupCRUD",
+    "GroupResourceScopedCRUD",
+    "HostedExtractorDestinationCRUD",
+    "HostedExtractorJobCRUD",
+    "HostedExtractorMappingCRUD",
+    "HostedExtractorSourceCRUD",
+    "InFieldCDMLocationConfigCRUD",
+    "InFieldLocationConfigCRUD",
+    "LabelCRUD",
+    "LocationFilterCRUD",
+    "NodeCRUD",
+    "RawDatabaseCRUD",
+    "RawFileCRUD",
+    "RawTableCRUD",
+    "RelationshipCRUD",
+    "ResourceCRUD",
+    "ResourceContainerCRUD",
+    "ResourceTypes",
+    "ResourceWorker",
+    "RobotCapabilityCRUD",
+    "RoboticFrameCRUD",
+    "RoboticLocationCRUD",
+    "RoboticMapCRUD",
+    "RoboticsDataPostProcessingCRUD",
+    "SearchConfigCRUD",
+    "SecurityCategoryCRUD",
+    "SequenceCRUD",
+    "SequenceRowCRUD",
+    "SpaceCRUD",
+    "StreamlitCRUD",
+    "ThreeDModelCRUD",
+    "TimeSeriesCRUD",
+    "TransformationCRUD",
+    "TransformationNotificationCRUD",
+    "TransformationScheduleCRUD",
+    "ViewCRUD",
+    "WorkflowCRUD",
+    "WorkflowTriggerCRUD",
+    "WorkflowVersionCRUD",
+    "get_crud",
+]