cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py

@@ -15,7 +15,6 @@ from rich import print
 from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_ENVIRONMENT_FILE,
-    BUILTIN_MODULES,
     DEFAULT_CONFIG_FILE,
     DEFAULT_ENV,
     MODULES,
@@ -468,7 +467,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
         """Loads all default.config.yaml files in the cognite root module."""
 
         default_files_iterable: Iterable[Path]
-        if cognite_root_module.name in ROOT_MODULES or cognite_root_module.name == BUILTIN_MODULES:
+        if cognite_root_module.name in ROOT_MODULES:
             default_files_iterable = cognite_root_module.glob(f"**/{DEFAULT_CONFIG_FILE}")
         else:
             default_files_iterable = itertools.chain(
@@ -497,7 +496,9 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
         adds them to the config.yaml file.
 
         Args:
-            cognite_root_module: The root module for all cognite modules.
+            cognite_root_module: Path to the root directory containing all Cognite modules.
+            defaults_files: List of paths to default.config.yaml files to load.
+            ignore_patterns: Optional list of tuples containing patterns to ignore when loading defaults.
 
         Returns:
             self
@@ -510,6 +511,10 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
             raw_file = safe_read(default_config)
             file_comments = self._extract_comments(raw_file, key_prefix=tuple(parts))
             file_data = cast(dict, read_yaml_content(raw_file))
+
+            # a file may exist, but contain just comments, thus the file_data is None
+            if file_data is None:
+                continue
             for key, value in file_data.items():
                 if len(parts) >= 1 and parts[0] in ROOT_MODULES:
                     key_path = (self._variables, *parts, key)

cognite_toolkit/_cdf_tk/data_classes/_module_directories.py

@@ -50,6 +50,20 @@ class ModuleLocation:
             return self.definition.title
         return None
 
+    @property
+    def module_id(self) -> str | None:
+        """The ID of the module."""
+        if self.definition:
+            return self.definition.id
+        return None
+
+    @property
+    def package_id(self) -> str | None:
+        """The ID of the package."""
+        if self.definition:
+            return self.definition.package_id
+        return None
+
     @property
     def relative_path(self) -> Path:
         """The relative path to the module."""
@@ -150,6 +164,8 @@ class ModuleLocation:
         return ReadModule(
             dir=self.dir,
             resource_directories=tuple(self.resource_directories),
+            module_id=self.module_id,
+            package_id=self.package_id,
         )
 
 
@@ -164,6 +180,8 @@ class ReadModule:
 
     dir: Path
     resource_directories: tuple[str, ...]
+    module_id: str | None
+    package_id: str | None
 
     def resource_dir_path(self, resource_folder: str) -> Path | None:
         """Returns the path to a resource in the module.
@@ -184,12 +202,16 @@ class ReadModule:
         return cls(
             dir=Path(data["dir"]),
             resource_directories=tuple(data["resource_directories"]),
+            module_id=data.get("module_id"),
+            package_id=data.get("package_id"),
         )
 
     def dump(self) -> dict[str, Any]:
         return {
             "dir": self.dir.as_posix(),
             "resource_directories": list(self.resource_directories),
+            "module_id": self.module_id,
+            "package_id": self.package_id,
         }
 
 

cognite_toolkit/_cdf_tk/data_classes/_module_toml.py

@@ -35,10 +35,12 @@ class ExampleData:
 class ModuleToml:
     filename: ClassVar[str] = "module.toml"
     title: str | None
+    id: str | None = None
     dependencies: frozenset[str] = field(default_factory=frozenset)
     is_selected_by_default: bool = False
     data: list[ExampleData] = field(default_factory=list)
     extra_resources: list[Path] = field(default_factory=list)
+    package_id: str | None = None
 
     def __post_init__(self) -> None:
         for extra in self.extra_resources:
@@ -64,15 +66,20 @@ class ModuleToml:
             extra_resources = [Path(item["location"]) for item in data["extra_resources"] if "location" in item]
 
         title: str | None = None
+        id: str | None = None
         is_selected_by_default: bool = False
         if "module" in data:
             title = data["module"].get("title")
+            id = data["module"].get("id")
             is_selected_by_default = data["module"].get("is_selected_by_default", False)
+            package_id = data["module"].get("package_id")
 
         return cls(
             title=title,
+            id=id,
             dependencies=dependencies,
             is_selected_by_default=is_selected_by_default,
             data=example_data,
             extra_resources=extra_resources,
+            package_id=package_id,
         )

cognite_toolkit/_cdf_tk/data_classes/_packages.py

@@ -3,7 +3,9 @@ from collections.abc import ItemsView, Iterable, Iterator, KeysView, Mapping, Mu
 from dataclasses import dataclass, field
 from pathlib import Path
 
-from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitValueError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError
+from cognite_toolkit._cdf_tk.tk_warnings.base import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.tk_warnings.other import LowSeverityWarning
 
 from ._module_directories import ModuleDirectories, ModuleLocation
 
@@ -23,12 +25,14 @@ class Package:
         name: the unique identifier of the package.
         title: The display name of the package.
         description: A description of the package.
+        id: An optional identifier for package statistics.
         modules: The modules that are part of the package.
     """
 
     name: str
     title: str
     description: str | None = None
+    id: str | None = None
     can_cherry_pick: bool = True
     modules: list[ModuleLocation] = field(default_factory=list)
 
@@ -43,12 +47,19 @@ class Package:
             name=name,
             title=package_definition["title"],
             description=package_definition.get("description"),
+            id=package_definition.get("id"),
             can_cherry_pick=package_definition.get("canCherryPick", True),
         )
 
 
 class Packages(dict, MutableMapping[str, Package]):
-    def __init__(self, packages: Iterable[Package] | Mapping[str, Package] | None = None) -> None:
+    warnings: WarningList[ToolkitWarning] | None = None
+
+    def __init__(
+        self,
+        packages: Iterable[Package] | Mapping[str, Package] | None = None,
+        warnings: WarningList[ToolkitWarning] | None = None,
+    ) -> None:
         if packages is None:
             super().__init__()
         elif isinstance(packages, Mapping):
@@ -56,6 +67,9 @@ class Packages(dict, MutableMapping[str, Package]):
         else:
             super().__init__({p.name: p for p in packages})
 
+        if warnings:
+            self.warnings = warnings
+
     @classmethod
     def load(
         cls,
@@ -82,20 +96,23 @@ class Packages(dict, MutableMapping[str, Package]):
 
         packages_with_modules: dict[str, Package] = {}
 
+        warnings = WarningList[ToolkitWarning]()
+
         for package_name, package_definition in package_definitions.items():
             packages_with_modules[package_name] = Package.load(package_name, package_definition)
             if modules := package_definition.get("modules"):
                 if isinstance(modules, list) and modules:
                     for module_path in modules:
                         if (module := module_by_relative_path.get(Path(module_path))) is None:
-                            available = sorted(str(m.relative_path) for m in module_directories)
-                            raise ToolkitValueError(
-                                f"Module '{module_path}' not found in the module directories.\n"
-                                f"Available modules: {available}"
+                            warnings.append(
+                                LowSeverityWarning(
+                                    f"Unable to load module '{module_path}'. The path may be wrong or the module may require an alpha flag that is not set."
+                                )
                             )
+                            continue
                         packages_with_modules[package_name].modules.append(module)
 
-        return cls(packages_with_modules)
+        return cls(packages_with_modules, warnings)
 
     # The methods are overloads to provide type hints for the methods.
     def items(self) -> ItemsView[str, Package]:  # type: ignore[override]

cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py

@@ -0,0 +1,43 @@
+"""Data class for command tracking information."""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class CommandTrackingInfo(BaseModel):
+    """Structured tracking information for CLI commands.
+
+    This model provides type-safe tracking information that can be collected
+    during command execution and sent to Mixpanel for analytics.
+
+    Attributes:
+        project: The CDF project name.
+        cluster: The CDF cluster name.
+        module_ids: List of module IDs that were deployed or built.
+        package_ids: List of package IDs that were deployed or built.
+        installed_module_ids: List of module IDs that were installed.
+        installed_package_ids: List of package IDs that were installed.
+        downloaded_library_ids: List of library IDs that were downloaded.
+        downloaded_package_ids: List of package IDs that were downloaded.
+        downloaded_module_ids: List of module IDs that were downloaded.
+    """
+
+    project: str | None = Field(default=None)
+    cluster: str | None = Field(default=None)
+    module_ids: set[str] = Field(default_factory=set, alias="moduleIds")
+    package_ids: set[str] = Field(default_factory=set, alias="packageIds")
+    installed_module_ids: set[str] = Field(default_factory=set, alias="installedModuleIds")
+    installed_package_ids: set[str] = Field(default_factory=set, alias="installedPackageIds")
+    downloaded_library_ids: set[str] = Field(default_factory=set, alias="downloadedLibraryIds")
+    downloaded_package_ids: set[str] = Field(default_factory=set, alias="downloadedPackageIds")
+    downloaded_module_ids: set[str] = Field(default_factory=set, alias="downloadedModuleIds")
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert the tracking info to a dictionary for Mixpanel.
+
+        Returns:
+            A dictionary with camelCase keys matching Mixpanel's expected format.
+            Default values are excluded.
+        """
+        return self.model_dump(mode="json", by_alias=True, exclude_defaults=True)

cognite_toolkit/_cdf_tk/feature_flags.py

@@ -22,25 +22,13 @@ class Flags(Enum):
         description="Enables the import sub application",
     )
     GRAPHQL = FlagMetadata(
-        visible=True,
+        visible=False,
         description="Enables the support for deploying data models as GraphQL schemas",
     )
     MODULE_REPEAT = FlagMetadata(
         visible=True,
         description="Enables the support for repeating modules in the config file",
     )
-    POPULATE = FlagMetadata(
-        visible=True,
-        description="Enables support for the populate command",
-    )
-    AGENTS = FlagMetadata(
-        visible=True,
-        description="Enables support for Atlas Agents and Agent Tools",
-    )
-    DUMP_EXTENDED = FlagMetadata(
-        visible=True,
-        description="Enables support for the dumping Location Filters.",
-    )
     PROFILE = FlagMetadata(
         visible=True,
         description="Enables support for the profile command",
@@ -49,37 +37,29 @@ class Flags(Enum):
         visible=True,
         description="Enables support for Infield configs",
     )
-    DUMP_DATA = FlagMetadata(
-        visible=True,
-        description="Splits the dump command in dump data and dump config",
-    )
-    EXIT_ON_WARNING = FlagMetadata(
-        visible=True,
-        description="Enables the exit on warning feature during the build command",
-    )
     MIGRATE = FlagMetadata(
         visible=True,
         description="Enables the migrate command",
     )
-    EXTERNAL_LIBRARIES = FlagMetadata(
+    STREAMS = FlagMetadata(
         visible=True,
-        description="Enables the support for external libraries in the config file",
+        description="Enables the support for the streams resources",
     )
-    PURGE_INSTANCES = FlagMetadata(
-        visible=True,
-        description="Enables the cdf purge instances command",
+    v08 = FlagMetadata(
+        visible=False,
+        description="Enables features planned for Cognite Toolkit version 0.8.0",
     )
-    DOWNLOAD = FlagMetadata(
+    CREATE = FlagMetadata(
         visible=True,
-        description="Enables the support for the download command",
+        description="Enables the support for the resources create command under dev plugin",
     )
-    UPLOAD = FlagMetadata(
+    EXTEND_DOWNLOAD = FlagMetadata(
         visible=True,
-        description="Enables the cdf upload command",
+        description="Enables extended download to support downloading file content and datapoints",
     )
-    SEARCH_CONFIG = FlagMetadata(
+    EXTEND_UPLOAD = FlagMetadata(
         visible=True,
-        description="Enables the support for the search config command",
+        description="Enables extended upload to support uploading individual files",
     )
 
     def is_enabled(self) -> bool:
@@ -91,3 +71,7 @@ class FeatureFlag:
     @lru_cache(typed=True)
     def is_enabled(flag: Flags) -> bool:
         return CDFToml.load().alpha_flags.get(clean_name(flag.name), False)
+
+    @staticmethod
+    def flush() -> None:
+        FeatureFlag.is_enabled.cache_clear()

cognite_toolkit/_cdf_tk/plugins.py

@@ -17,7 +17,8 @@ class Plugin:
 class Plugins(Enum):
     run = Plugin("run", "plugin for Run command to execute Python scripts in CDF")
     dump = Plugin("dump", "plugin for Dump command to retrieve Asset resources from CDF")
-    purge = Plugin("purge", "plugin for Purge command to remove datasets and spaces from CDF")
+    dev = Plugin("dev", "plugin for commands to develop modules in CDF")
+    data = Plugin("data", "plugin for Data command to manage data in CDF")
 
     @staticmethod
     def list() -> dict[str, bool]:

cognite_toolkit/_cdf_tk/protocols.py

@@ -0,0 +1,97 @@
+import sys
+from collections.abc import Collection, Iterator
+from typing import Any, Generic, Protocol, TypeVar
+
+from cognite.client import CogniteClient
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class ResourceRequestProtocol(Protocol):
+    @classmethod
+    def _load(cls, data: dict[str, Any]) -> Self: ...
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]: ...
+
+
+class ResourceResponseProtocol(Protocol):
+    def as_write(self) -> ResourceRequestProtocol: ...
+
+
+T_ResourceRequest = TypeVar("T_ResourceRequest", bound=ResourceRequestProtocol)
+T_ResourceResponse = TypeVar("T_ResourceResponse", bound=ResourceResponseProtocol)
+
+
+class ResourceRequestListProtocol(Protocol, Generic[T_ResourceRequest]):
+    def __init__(self, collection: Collection[T_ResourceRequest]): ...
+
+    def __getitem__(self, index: int) -> T_ResourceRequest: ...
+
+    def __setitem__(self, index: int, value: T_ResourceRequest) -> None: ...
+
+    def __delitem__(self, index: int) -> None: ...
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[T_ResourceRequest]: ...
+
+    def __contains__(self, value: object) -> bool: ...
+
+    def __reversed__(self) -> Iterator[T_ResourceRequest]: ...
+
+    def insert(self, index: int, value: T_ResourceRequest) -> None: ...
+
+    def append(self, value: T_ResourceRequest) -> None: ...
+
+    def extend(self, values: Collection[T_ResourceRequest]) -> None: ...
+
+    def pop(self, index: int = -1) -> T_ResourceRequest: ...
+
+    def remove(self, value: T_ResourceRequest) -> None: ...
+
+    def clear(self) -> None: ...
+
+    @classmethod
+    def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> Self: ...
+
+
+class ResourceResponseListProtocol(Protocol, Generic[T_ResourceResponse]):
+    def __init__(self, collection: Collection[T_ResourceResponse]): ...
+
+    def __getitem__(self, index: int) -> T_ResourceResponse: ...
+
+    def __setitem__(self, index: int, value: T_ResourceResponse) -> None: ...
+
+    def __delitem__(self, index: int) -> None: ...
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[T_ResourceResponse]: ...
+
+    def __contains__(self, value: object) -> bool: ...
+
+    def __reversed__(self) -> Iterator[T_ResourceResponse]: ...
+
+    def insert(self, index: int, value: T_ResourceResponse) -> None: ...
+
+    def append(self, value: T_ResourceResponse) -> None: ...
+
+    def extend(self, values: Collection[T_ResourceResponse]) -> None: ...
+
+    def pop(self, index: int = -1) -> T_ResourceResponse: ...
+
+    def remove(self, value: T_ResourceResponse) -> None: ...
+
+    def clear(self) -> None: ...
+
+    def as_write(self) -> ResourceRequestListProtocol: ...
+
+    @classmethod
+    def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> Self: ...
+
+
+T_ResourceRequestList = TypeVar("T_ResourceRequestList", bound=ResourceRequestListProtocol)
+T_ResourceResponseList = TypeVar("T_ResourceResponseList", bound=ResourceResponseListProtocol)

cognite_toolkit/_cdf_tk/resource_classes/__init__.py

@@ -7,8 +7,10 @@ This is means that we have three set of resource classes we use in Toolkit:
 3. Read resource classes (from cognite-sdk): Represent the read/response format the Cognite resources.
 """
 
+from .agent import AgentYAML
 from .asset import AssetYAML
 from .base import BaseModelResource, ToolkitResource
+from .cognitefile import CogniteFileYAML
 from .containers import ContainerYAML
 from .data_model import DataModelYAML
 from .datapoint_subscription import DatapointSubscriptionYAML
@@ -19,22 +21,28 @@ from .extraction_pipeline_config import ExtractionPipelineConfigYAML
 from .filemetadata import FileMetadataYAML
 from .function_schedule import FunctionScheduleYAML
 from .functions import FunctionsYAML
+from .graphql_model import GraphQLDataModelYAML
 from .groups import GroupYAML
 from .hosted_extractor_destination import HostedExtractorDestinationYAML
 from .hosted_extractor_job import HostedExtractorJobYAML
 from .hosted_extractor_mapping import HostedExtractorMappingYAML
 from .hosted_extractor_source import HostedExtractorSourceYAML
+from .infield_cdm_location_config import InFieldCDMLocationConfigYAML
+from .infield_location_config import InfieldLocationConfigYAML
+from .infield_v1 import InfieldV1YAML
 from .instance import EdgeYAML, NodeYAML
 from .labels import LabelsYAML
 from .location import LocationYAML
+from .migration import ResourceViewMappingYAML
 from .raw_database_table import DatabaseYAML, TableYAML
 from .relationship import RelationshipYAML
 from .robotics import RobotCapabilityYAML, RobotDataPostProcessingYAML, RobotFrameYAML, RobotLocationYAML, RobotMapYAML
 from .search_config import SearchConfigYAML
 from .securitycategories import SecurityCategoriesYAML
-from .sequence import SequenceYAML
+from .sequence import SequenceRowYAML, SequenceYAML
 from .space import SpaceYAML
 from .streamlit_ import StreamlitYAML
+from .streams import StreamYAML
 from .threedmodels import ThreeDModelYAML
 from .timeseries import TimeSeriesYAML
 from .transformation_notification import TransformationNotificationYAML
@@ -46,8 +54,10 @@ from .workflow_trigger import WorkflowTriggerYAML
 from .workflow_version import WorkflowVersionYAML
 
 __all__ = [
+    "AgentYAML",
     "AssetYAML",
     "BaseModelResource",
+    "CogniteFileYAML",
     "ContainerYAML",
     "DataModelYAML",
     "DataModelYAML",
@@ -61,15 +71,20 @@ __all__ = [
     "FileMetadataYAML",
     "FunctionScheduleYAML",
     "FunctionsYAML",
+    "GraphQLDataModelYAML",
     "GroupYAML",
     "HostedExtractorDestinationYAML",
     "HostedExtractorJobYAML",
     "HostedExtractorMappingYAML",
     "HostedExtractorSourceYAML",
+    "InFieldCDMLocationConfigYAML",
+    "InfieldLocationConfigYAML",
+    "InfieldV1YAML",
     "LabelsYAML",
     "LocationYAML",
     "NodeYAML",
     "RelationshipYAML",
+    "ResourceViewMappingYAML",
     "RobotCapabilityYAML",
     "RobotDataPostProcessingYAML",
     "RobotFrameYAML",
@@ -77,8 +92,10 @@ __all__ = [
     "RobotMapYAML",
     "SearchConfigYAML",
     "SecurityCategoriesYAML",
+    "SequenceRowYAML",
     "SequenceYAML",
     "SpaceYAML",
+    "StreamYAML",
     "StreamlitYAML",
     "TableYAML",
     "ThreeDModelYAML",

cognite_toolkit/_cdf_tk/resource_classes/agent.py

@@ -0,0 +1,58 @@
+from typing import Literal
+
+from pydantic import Field
+
+from .agent_tools import AgentTool
+from .base import ToolkitResource
+
+Model = Literal[
+    "azure/o3",
+    "azure/o4-mini",
+    "azure/gpt-4o",
+    "azure/gpt-4o-mini",
+    "azure/gpt-4.1",
+    "azure/gpt-4.1-nano",
+    "azure/gpt-4.1-mini",
+    "azure/gpt-5",
+    "azure/gpt-5-mini",
+    "azure/gpt-5-nano",
+    "gcp/gemini-2.5-pro",
+    "gcp/gemini-2.5-flash",
+    "aws/claude-4-sonnet",
+    "aws/claude-4-opus",
+    "aws/claude-4.1-opus",
+    "aws/claude-3.5-sonnet",
+]
+
+
+class AgentYAML(ToolkitResource):
+    """Atlas AI Agent"""
+
+    external_id: str = Field(
+        description="An external ID that uniquely identifies the agent.",
+        min_length=1,
+        max_length=128,
+        pattern=r"^[^\x00]{1,128}$",
+    )
+    name: str = Field(
+        description="A descriptive name intended for use in user interfaces.",
+        min_length=1,
+        max_length=255,
+    )
+    description: str | None = Field(
+        default=None,
+        description="A human-readable description of what the agent does, used for documentation only. "
+        "This description is not used by the language model.",
+        max_length=1024,
+    )
+    instructions: str | None = Field(
+        default=None,
+        description="The instructions for the agent prompt the language model to understand "
+        "the agent's goals and how to achieve them.",
+        max_length=32000,
+    )
+    model: Model = Field(
+        "azure/gpt-4o-mini", description="The name of the model to use. Defaults to your CDF project's default model."
+    )
+    tools: list[AgentTool] | None = Field(None, description="A list of tools available to the agent.", max_length=20)
+    runtime_version: str | None = Field(None, description="The runtime version")