cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.39__py3-none-any.whl

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py

@@ -26,6 +26,8 @@
 # limitations under the License.
 
 
+import random
+import time
 import warnings
 from collections import defaultdict
 from collections.abc import Callable, Hashable, Iterable, Sequence
@@ -43,9 +45,7 @@ from cognite.client.data_classes import (
     TransformationSchedule,
     TransformationScheduleList,
     TransformationScheduleWrite,
-    TransformationScheduleWriteList,
     TransformationWrite,
-    TransformationWriteList,
 )
 from cognite.client.data_classes.capabilities import (
     Capability,
@@ -58,7 +58,6 @@ from cognite.client.data_classes.data_modeling.ids import (
 from cognite.client.data_classes.transformations import NonceCredentials
 from cognite.client.data_classes.transformations.notifications import (
     TransformationNotificationWrite,
-    TransformationNotificationWriteList,
 )
 from cognite.client.exceptions import CogniteAPIError, CogniteAuthError, CogniteDuplicatedError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
@@ -66,7 +65,7 @@ from rich import print
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.raw import RawDatabase, RawTable
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawDatabase, RawTable
 from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
@@ -82,7 +81,7 @@ from cognite_toolkit._cdf_tk.resource_classes import (
     TransformationScheduleYAML,
     TransformationYAML,
 )
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils import (
     calculate_secure_hash,
     humanize_collection,
@@ -103,18 +102,10 @@ from .raw import RawDatabaseCRUD, RawTableCRUD
 
 
 @final
-class TransformationCRUD(
-    ResourceCRUD[str, TransformationWrite, Transformation, TransformationWriteList, TransformationList]
-):
+class TransformationCRUD(ResourceCRUD[str, TransformationWrite, Transformation]):
     folder_name = "transformations"
-    filename_pattern = (
-        # Matches all yaml files except file names whose stem contain *.schedule. or .Notification
-        r"^(?!.*schedule.*|.*\.notification$).*$"
-    )
     resource_cls = Transformation
     resource_write_cls = TransformationWrite
-    list_cls = TransformationList
-    list_write_cls = TransformationWriteList
     kind = "Transformation"
     yaml_cls = TransformationYAML
     dependencies = frozenset(
@@ -354,6 +345,16 @@ class TransformationCRUD(
         if "authentication" in local:
             # The hash added to the beginning of the query detects the change in the authentication
             dumped["authentication"] = local["authentication"]
+        cdf_destination = dumped.get("destination", {})
+        local_destination = local.get("destination", {})
+        if isinstance(cdf_destination, dict) and isinstance(local_destination, dict):
+            if cdf_destination.get("instanceSpace") is None and "instanceSpace" not in local_destination:
+                cdf_destination.pop("instanceSpace", None)
+        if not dumped.get("query") and "query" not in local:
+            dumped.pop("query", None)
+        if dumped.get("conflictMode") == "upsert" and "conflictMode" not in local:
+            # Default set from server side.
+            dumped.pop("conflictMode", None)
         return dumped
 
     def split_resource(
@@ -427,16 +428,70 @@ class TransformationCRUD(
                 if error := self._create_auth_creation_error(chunk):
                     raise error from e
                 raise e
-            results.extend(chunk_results)
+            except CogniteAPIError as e:
+                if "Failed to bind session using nonce" in e.message and len(chunk) > 1:
+                    results.extend(self._execute_one_by_one(chunk, api_call))
+                else:
+                    raise
+            else:
+                results.extend(chunk_results)
+        return results
+
+    def _execute_one_by_one(
+        self,
+        chunk: Sequence[TransformationWrite],
+        api_call: Callable[[Sequence[TransformationWrite]], TransformationList],
+    ) -> TransformationList:
+        MediumSeverityWarning(
+            f"Failed to create {len(chunk)} transformations in a batch due to nonce binding error. "
+            "Trying to recover by creating them one by one."
+        ).print_warning(console=self.client.console)
+        # Retry one by one
+        failed_ids: list[str] = []
+        success_count = 0
+        delay = 0.3
+        self._sleep_with_jitter(delay, delay + 0.3)
+        results = TransformationList([])
+        for item in chunk:
+            try:
+                recovered = api_call([item])
+            except CogniteAPIError as e:
+                if "Failed to bind session using nonce" in e.message:
+                    failed_ids.append(item.external_id or "<missing>")
+                    self._sleep_with_jitter(delay, delay + 0.3)
+                else:
+                    raise
+            else:
+                results.extend(recovered)
+                success_count += 1
+        message = f"  [bold]RECOVERY COMPLETE:[/] Successfully created {success_count:,} transformations"
+        if failed_ids:
+            message += f", failed to create {len(failed_ids):,} transformations: {humanize_collection(failed_ids)}"
+        else:
+            message += "."
+        if failed_ids:
+            HighSeverityWarning(message).print_warning(include_timestamp=True, console=self.client.console)
+        else:
+            self.client.console.print(message)
         return results
 
+    @staticmethod
+    def _sleep_with_jitter(base_delay: float, max_delay: float) -> None:
+        """Sleeps for a random duration between base_delay and max_delay (inclusive)."""
+        sleep_time = random.uniform(base_delay, max_delay)
+        time.sleep(sleep_time)
+
     def _update_nonce(self, items: Sequence[TransformationWrite]) -> None:
         for item in items:
             if not item.external_id:
                 raise ToolkitRequiredValueError("Transformation must have external_id set.")
-            if read_credentials := self._authentication_by_id_operation.get((item.external_id, "read")):
+            if item.source_nonce is None and (
+                read_credentials := self._authentication_by_id_operation.get((item.external_id, "read"))
+            ):
                 item.source_nonce = self._create_nonce(read_credentials)
-            if write_credentials := self._authentication_by_id_operation.get((item.external_id, "write")):
+            if item.destination_nonce is None and (
+                write_credentials := self._authentication_by_id_operation.get((item.external_id, "write"))
+            ):
                 item.destination_nonce = self._create_nonce(write_credentials)
 
     def _create_nonce(self, credentials: OidcCredentials | ClientCredentials) -> NonceCredentials:
@@ -483,17 +538,11 @@ class TransformationScheduleCRUD(
         str,
         TransformationScheduleWrite,
         TransformationSchedule,
-        TransformationScheduleWriteList,
-        TransformationScheduleList,
     ]
 ):
     folder_name = "transformations"
-    # Matches all yaml files whose stem contains *schedule or *TransformationSchedule.
-    filename_pattern = r"^.*schedule$"
     resource_cls = TransformationSchedule
     resource_write_cls = TransformationScheduleWrite
-    list_cls = TransformationScheduleList
-    list_write_cls = TransformationScheduleWriteList
     kind = "Schedule"
     yaml_cls = TransformationScheduleYAML
     dependencies = frozenset({TransformationCRUD})
@@ -543,7 +592,7 @@ class TransformationScheduleCRUD(
     def retrieve(self, ids: SequenceNotStr[str]) -> TransformationScheduleList:
         return self.client.transformations.schedules.retrieve_multiple(external_ids=ids, ignore_unknown_ids=True)
 
-    def update(self, items: TransformationScheduleWriteList) -> TransformationScheduleList:
+    def update(self, items: Sequence[TransformationScheduleWrite]) -> TransformationScheduleList:
         return self.client.transformations.schedules.update(items, mode="replace")
 
     def delete(self, ids: str | SequenceNotStr[str] | None) -> int:
@@ -581,17 +630,11 @@ class TransformationNotificationCRUD(
         str,
         TransformationNotificationWrite,
         TransformationNotification,
-        TransformationNotificationWriteList,
-        TransformationNotificationList,
     ]
 ):
     folder_name = "transformations"
-    # Matches all yaml files whose stem ends with *Notification.
-    filename_pattern = r"^.*Notification$"
     resource_cls = TransformationNotification
     resource_write_cls = TransformationNotificationWrite
-    list_cls = TransformationNotificationList
-    list_write_cls = TransformationNotificationWriteList
     kind = "Notification"
     dependencies = frozenset({TransformationCRUD})
     _doc_url = "Transformation-Notifications/operation/createTransformationNotifications"
@@ -638,7 +681,7 @@ class TransformationNotificationCRUD(
             dumped["transformationExternalId"] = local["transformationExternalId"]
         return dumped
 
-    def create(self, items: TransformationNotificationWriteList) -> TransformationNotificationList:
+    def create(self, items: Sequence[TransformationNotificationWrite]) -> TransformationNotificationList:
         return self.client.transformations.notifications.create(items)
 
     def retrieve(self, ids: SequenceNotStr[str]) -> TransformationNotificationList:
@@ -669,7 +712,7 @@ class TransformationNotificationCRUD(
 
         return retrieved
 
-    def update(self, items: TransformationNotificationWriteList) -> TransformationNotificationList:
+    def update(self, items: Sequence[TransformationNotificationWrite]) -> TransformationNotificationList:
         # Note that since a notification is identified by the combination of transformationExternalId and destination,
         # which is the entire object, an update should never happen. However, implementing just in case.
         item_by_id = {self.get_id(item): item for item in items}

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py

@@ -22,24 +22,23 @@ from typing import Any, final
 from cognite.client.data_classes import (
     ClientCredentials,
     Workflow,
-    WorkflowList,
     WorkflowTrigger,
-    WorkflowTriggerList,
     WorkflowTriggerUpsert,
-    WorkflowTriggerUpsertList,
     WorkflowUpsert,
-    WorkflowUpsertList,
     WorkflowVersion,
     WorkflowVersionId,
-    WorkflowVersionList,
     WorkflowVersionUpsert,
-    WorkflowVersionUpsertList,
 )
 from cognite.client.data_classes.capabilities import (
     Capability,
     WorkflowOrchestrationAcl,
 )
-from cognite.client.data_classes.workflows import SubworkflowReferenceParameters
+from cognite.client.data_classes.workflows import (
+    SubworkflowReferenceParameters,
+    WorkflowList,
+    WorkflowTriggerList,
+    WorkflowVersionList,
+)
 from cognite.client.exceptions import CogniteAuthError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich import print
@@ -75,13 +74,10 @@ from .transformation import TransformationCRUD
 
 
 @final
-class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertList, WorkflowList]):
+class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow]):
     folder_name = "workflows"
-    filename_pattern = r"^.*Workflow$"
     resource_cls = Workflow
     resource_write_cls = WorkflowUpsert
-    list_cls = WorkflowList
-    list_write_cls = WorkflowUpsertList
     kind = "Workflow"
     dependencies = frozenset(
         {
@@ -140,18 +136,14 @@ class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertLis
             dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
         return dumped
 
-    def retrieve(self, ids: SequenceNotStr[str]) -> WorkflowList:
+    def retrieve(self, ids: SequenceNotStr[str]) -> Sequence[Workflow]:
         return self.client.workflows.retrieve(external_id=ids, ignore_unknown_ids=True)
 
-    def _upsert(self, items: WorkflowUpsert | WorkflowUpsertList) -> WorkflowList:
-        upserts = [items] if isinstance(items, WorkflowUpsert) else items
-        return self.client.workflows.upsert(upserts)
+    def create(self, items: Sequence[WorkflowUpsert]) -> WorkflowList:
+        return self.client.workflows.upsert(items)
 
-    def create(self, items: WorkflowUpsert | WorkflowUpsertList) -> WorkflowList:
-        return self._upsert(items)
-
-    def update(self, items: WorkflowUpsertList) -> WorkflowList:
-        return self._upsert(items)
+    def update(self, items: Sequence[WorkflowUpsert]) -> WorkflowList:
+        return self.client.workflows.upsert(items)
 
     def delete(self, ids: SequenceNotStr[str]) -> int:
         successes = 0
@@ -192,17 +184,10 @@ class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertLis
 
 
 @final
-class WorkflowVersionCRUD(
-    ResourceCRUD[
-        WorkflowVersionId, WorkflowVersionUpsert, WorkflowVersion, WorkflowVersionUpsertList, WorkflowVersionList
-    ]
-):
+class WorkflowVersionCRUD(ResourceCRUD[WorkflowVersionId, WorkflowVersionUpsert, WorkflowVersion]):
     folder_name = "workflows"
-    filename_pattern = r"^.*WorkflowVersion$"
     resource_cls = WorkflowVersion
     resource_write_cls = WorkflowVersionUpsert
-    list_cls = WorkflowVersionList
-    list_write_cls = WorkflowVersionUpsertList
     kind = "WorkflowVersion"
     dependencies = frozenset({WorkflowCRUD})
     parent_resource = frozenset({WorkflowCRUD})
@@ -398,16 +383,16 @@ class WorkflowVersionCRUD(
             return WorkflowVersionList([])
         return self.client.workflows.versions.retrieve(workflow_external_id=list(ids), ignore_unknown_ids=True)
 
-    def _upsert(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList:
+    def _upsert(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
         return self.client.workflows.versions.upsert(items)
 
-    def create(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList:
+    def create(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
         upserted: list[WorkflowVersion] = []
         for item in self.topological_sort(items):
             upserted.extend(self.client.workflows.versions.upsert([item]))
         return WorkflowVersionList(upserted)
 
-    def update(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList:
+    def update(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
         return self._upsert(items)
 
     def delete(self, ids: SequenceNotStr[WorkflowVersionId]) -> int:
@@ -467,15 +452,10 @@ class WorkflowVersionCRUD(
 
 
 @final
-class WorkflowTriggerCRUD(
-    ResourceCRUD[str, WorkflowTriggerUpsert, WorkflowTrigger, WorkflowTriggerUpsertList, WorkflowTriggerList]
-):
+class WorkflowTriggerCRUD(ResourceCRUD[str, WorkflowTriggerUpsert, WorkflowTrigger]):
     folder_name = "workflows"
-    filename_pattern = r"^.*WorkflowTrigger$"
     resource_cls = WorkflowTrigger
     resource_write_cls = WorkflowTriggerUpsert
-    list_cls = WorkflowTriggerList
-    list_write_cls = WorkflowTriggerUpsertList
     kind = "WorkflowTrigger"
     dependencies = frozenset({WorkflowCRUD, WorkflowVersionCRUD, GroupResourceScopedCRUD, GroupAllScopedCRUD})
     parent_resource = frozenset({WorkflowCRUD})
@@ -522,10 +502,10 @@ class WorkflowTriggerCRUD(
             WorkflowOrchestrationAcl.Scope.All(),
         )
 
-    def create(self, items: WorkflowTriggerUpsertList) -> WorkflowTriggerList:
+    def create(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
         return self._upsert(items)
 
-    def _upsert(self, items: WorkflowTriggerUpsertList) -> WorkflowTriggerList:
+    def _upsert(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
         created = WorkflowTriggerList([])
         for item in items:
             created.append(self._upsert_item(item))
@@ -545,7 +525,7 @@ class WorkflowTriggerCRUD(
         lookup = set(ids)
         return WorkflowTriggerList([trigger for trigger in all_triggers if trigger.external_id in lookup])
 
-    def update(self, items: WorkflowTriggerUpsertList) -> WorkflowTriggerList:
+    def update(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
         return self._upsert(items)
 
     def delete(self, ids: SequenceNotStr[str]) -> int:

cognite_toolkit/_cdf_tk/cruds/_worker.py

@@ -1,8 +1,8 @@
 import re
 import warnings
-from collections.abc import Hashable
+from collections.abc import Hashable, Sequence
 from copy import deepcopy
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, cast
 
@@ -14,39 +14,29 @@ from yaml import YAMLError
 
 from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS
 from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError, ToolkitYAMLFormatError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings
 from cognite_toolkit._cdf_tk.utils import to_diff
 
 from . import FunctionCRUD
-from ._base_cruds import (
-    T_ID,
-    ResourceCRUD,
-    T_ResourceRequest,
-    T_ResourceRequestList,
-    T_ResourceResponse,
-    T_ResourceResponseList,
-)
+from ._base_cruds import T_ID, ResourceCRUD, T_ResourceRequest, T_ResourceResponse
 
 if TYPE_CHECKING:
     from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
 
 
 @dataclass
-class CategorizedResources(Generic[T_ID, T_ResourceRequestList]):
-    to_create: T_ResourceRequestList
-    to_update: T_ResourceRequestList
-    to_delete: list[T_ID]
-    unchanged: T_ResourceRequestList
+class CategorizedResources(Generic[T_ID, T_ResourceRequest]):
+    to_create: list[T_ResourceRequest] = field(default_factory=list)
+    to_update: list[T_ResourceRequest] = field(default_factory=list)
+    to_delete: list[T_ID] = field(default_factory=list)
+    unchanged: list[T_ResourceRequest] = field(default_factory=list)
 
 
-class ResourceWorker(
-    Generic[T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList]
-):
+class ResourceWorker(Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
     def __init__(
         self,
-        loader: ResourceCRUD[
-            T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         action: str,
     ):
         self.loader = loader
@@ -58,12 +48,16 @@ class ResourceWorker(
     ) -> list[Path]:
         filepaths = self.loader.find_files(directory)
 
-        for read_module in read_modules or []:
-            if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
-                # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
-                # These table formats are not built, i.e., no variable replacement is done,
-                # so we load them directly from the source module.
-                filepaths.extend(self.loader.find_files(resource_dir, include_formats=TABLE_FORMATS))
+        if not Flags.v08.is_enabled():
+            for read_module in read_modules or []:
+                if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
+                    # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
+                    # These table formats are not built, i.e., no variable replacement is done,
+                    # so we load them directly from the source module.
+                    table_files = [
+                        file for file in self.loader.find_files(resource_dir) if file.suffix in TABLE_FORMATS
+                    ]
+                    filepaths.extend(table_files)
 
         if not sort:
             return filepaths
@@ -104,7 +98,6 @@ class ResourceWorker(
         self.validate_access(local_by_id, is_dry_run)
 
         # Lookup the existing resources in CDF
-        cdf_resources: T_ResourceResponseList
         cdf_resources = self.loader.retrieve(list(local_by_id.keys()))
         return self.categorize_resources(local_by_id, cdf_resources, force_update, verbose)
 
@@ -165,16 +158,11 @@ class ResourceWorker(
     def categorize_resources(
         self,
         local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]],
-        cdf_resources: T_ResourceResponseList,
+        cdf_resources: Sequence[T_ResourceResponse],
         force_update: bool,
         verbose: bool,
-    ) -> CategorizedResources[T_ID, T_ResourceRequestList]:
-        resources: CategorizedResources[T_ID, T_ResourceRequestList] = CategorizedResources(
-            to_create=self.loader.list_write_cls([]),
-            to_update=self.loader.list_write_cls([]),
-            to_delete=[],
-            unchanged=self.loader.list_write_cls([]),
-        )
+    ) -> CategorizedResources[T_ID, T_ResourceRequest]:
+        resources: CategorizedResources[T_ID, T_ResourceRequest] = CategorizedResources()
         cdf_resource_by_id = {self.loader.get_id(resource): resource for resource in cdf_resources}
         for identifier, (local_dict, local_resource) in local_by_id.items():
             cdf_resource = cdf_resource_by_id.get(identifier)

cognite_toolkit/_cdf_tk/data_classes/_module_toml.py

@@ -68,6 +68,7 @@ class ModuleToml:
         title: str | None = None
         id: str | None = None
         is_selected_by_default: bool = False
+        package_id: str | None = None
         if "module" in data:
             title = data["module"].get("title")
             id = data["module"].get("id")

cognite_toolkit/_cdf_tk/feature_flags.py

@@ -22,21 +22,13 @@ class Flags(Enum):
         description="Enables the import sub application",
     )
     GRAPHQL = FlagMetadata(
-        visible=True,
+        visible=False,
         description="Enables the support for deploying data models as GraphQL schemas",
     )
     MODULE_REPEAT = FlagMetadata(
         visible=True,
         description="Enables the support for repeating modules in the config file",
     )
-    AGENTS = FlagMetadata(
-        visible=True,
-        description="Enables support for Atlas Agents and Agent Tools",
-    )
-    DUMP_EXTENDED = FlagMetadata(
-        visible=True,
-        description="Enables support for the dumping Location Filters.",
-    )
     PROFILE = FlagMetadata(
         visible=True,
         description="Enables support for the profile command",
@@ -45,45 +37,29 @@ class Flags(Enum):
         visible=True,
         description="Enables support for Infield configs",
     )
-    DUMP_DATA = FlagMetadata(
-        visible=True,
-        description="Splits the dump command in dump data and dump config",
-    )
-    EXIT_ON_WARNING = FlagMetadata(
-        visible=True,
-        description="Enables the exit on warning feature during the build command",
-    )
     MIGRATE = FlagMetadata(
         visible=True,
         description="Enables the migrate command",
     )
-    EXTERNAL_LIBRARIES = FlagMetadata(
-        visible=True,
-        description="Enables the support for external libraries in the config file",
-    )
-    PURGE_INSTANCES = FlagMetadata(
+    STREAMS = FlagMetadata(
         visible=True,
-        description="Enables the cdf purge instances command",
+        description="Enables the support for the streams resources",
     )
-    DOWNLOAD = FlagMetadata(
-        visible=True,
-        description="Enables the support for the download command",
+    v08 = FlagMetadata(
+        visible=False,
+        description="Enables features planned for Cognite Toolkit version 0.8.0",
     )
-    UPLOAD = FlagMetadata(
+    CREATE = FlagMetadata(
         visible=True,
-        description="Enables the cdf upload command",
+        description="Enables the support for the resources create command under dev plugin",
     )
-    SEARCH_CONFIG = FlagMetadata(
+    EXTEND_DOWNLOAD = FlagMetadata(
         visible=True,
-        description="Enables the support for the search config command",
+        description="Enables extended download to support downloading file content and datapoints",
     )
-    FUNCTION_COGNITE_FILE = FlagMetadata(
+    EXTEND_UPLOAD = FlagMetadata(
         visible=True,
-        description="Enables support for using Cognite Files to store function code",
-    )
-    v07 = FlagMetadata(
-        visible=False,
-        description="Enables features planned for Cognite Toolkit version 0.7.0",
+        description="Enables extended upload to support uploading individual files",
     )
 
     def is_enabled(self) -> bool:
@@ -95,3 +71,7 @@ class FeatureFlag:
     @lru_cache(typed=True)
     def is_enabled(flag: Flags) -> bool:
         return CDFToml.load().alpha_flags.get(clean_name(flag.name), False)
+
+    @staticmethod
+    def flush() -> None:
+        FeatureFlag.is_enabled.cache_clear()

cognite_toolkit/_cdf_tk/plugins.py

@@ -17,7 +17,8 @@ class Plugin:
 class Plugins(Enum):
     run = Plugin("run", "plugin for Run command to execute Python scripts in CDF")
     dump = Plugin("dump", "plugin for Dump command to retrieve Asset resources from CDF")
-    purge = Plugin("purge", "plugin for Purge command to remove datasets and spaces from CDF")
+    dev = Plugin("dev", "plugin for commands to develop modules in CDF")
+    data = Plugin("data", "plugin for Data command to manage data in CDF")
 
     @staticmethod
     def list() -> dict[str, bool]:

cognite_toolkit/_cdf_tk/resource_classes/__init__.py

@@ -27,6 +27,7 @@ from .hosted_extractor_destination import HostedExtractorDestinationYAML
 from .hosted_extractor_job import HostedExtractorJobYAML
 from .hosted_extractor_mapping import HostedExtractorMappingYAML
 from .hosted_extractor_source import HostedExtractorSourceYAML
+from .infield_cdm_location_config import InFieldCDMLocationConfigYAML
 from .infield_location_config import InfieldLocationConfigYAML
 from .infield_v1 import InfieldV1YAML
 from .instance import EdgeYAML, NodeYAML
@@ -41,6 +42,7 @@ from .securitycategories import SecurityCategoriesYAML
 from .sequence import SequenceRowYAML, SequenceYAML
 from .space import SpaceYAML
 from .streamlit_ import StreamlitYAML
+from .streams import StreamYAML
 from .threedmodels import ThreeDModelYAML
 from .timeseries import TimeSeriesYAML
 from .transformation_notification import TransformationNotificationYAML
@@ -75,6 +77,7 @@ __all__ = [
     "HostedExtractorJobYAML",
     "HostedExtractorMappingYAML",
     "HostedExtractorSourceYAML",
+    "InFieldCDMLocationConfigYAML",
     "InfieldLocationConfigYAML",
     "InfieldV1YAML",
     "LabelsYAML",
@@ -92,6 +95,7 @@ __all__ = [
     "SequenceRowYAML",
     "SequenceYAML",
     "SpaceYAML",
+    "StreamYAML",
     "StreamlitYAML",
     "TableYAML",
     "ThreeDModelYAML",

cognite_toolkit/_cdf_tk/resource_classes/capabilities.py

@@ -510,6 +510,18 @@ class LegacyGenericsAcl(Capability):
     scope: AllScope
 
 
+class StreamsAcl(Capability):
+    _capability_name = "streamsAcl"
+    actions: list[Literal["READ", "CREATE", "DELETE"]]
+    scope: AllScope
+
+
+class StreamRecordsAcl(Capability):
+    _capability_name = "streamRecordsAcl"
+    actions: list[Literal["READ", "WRITE"]]
+    scope: AllScope | SpaceIDScope
+
+
 _CAPABILITY_CLASS_BY_NAME: MappingProxyType[str, type[Capability]] = MappingProxyType(
     {c._capability_name: c for c in Capability.__subclasses__()}
 )

cognite_toolkit/_cdf_tk/resource_classes/functions.py

@@ -44,7 +44,9 @@ class FunctionsYAML(ToolkitResource):
     )
     cpu: float | None = Field(default=None, description="Number of CPU cores per function.")
     memory: float | None = Field(default=None, description="Memory per function measured in GB.")
-    runtime: Literal["py39", "py310", "py311"] | None = Field(default="py311", description="Runtime of the function.")
+    runtime: Literal["py39", "py310", "py311", "py312"] | None = Field(
+        default="py311", description="Runtime of the function."
+    )
     metadata: dict[str, str] | None = Field(
         default=None, description="Custom, application-specific metadata.", max_length=16
     )