cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/data_classes/_module_toml.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import sys
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -8,9 +6,12 @@ from typing import Any, ClassVar
 from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError
 
 if sys.version_info >= (3, 11):
+    from typing import Self
+
     import toml
 else:
     import tomli as toml
+    from typing_extensions import Self
 
 
 @dataclass(frozen=True)
@@ -21,7 +22,7 @@ class ExampleData:
     destination: Path
 
     @classmethod
-    def load(cls, data: dict[str, Any]) -> ExampleData:
+    def load(cls, data: dict[str, Any]) -> Self:
         return cls(
             repo_type=data["repoType"],
             repo=data["repo"],
@@ -34,11 +35,12 @@ class ExampleData:
 class ModuleToml:
     filename: ClassVar[str] = "module.toml"
     title: str | None
-    tags: frozenset[str] = field(default_factory=frozenset)
+    id: str | None = None
     dependencies: frozenset[str] = field(default_factory=frozenset)
     is_selected_by_default: bool = False
     data: list[ExampleData] = field(default_factory=list)
     extra_resources: list[Path] = field(default_factory=list)
+    package_id: str | None = None
 
     def __post_init__(self) -> None:
         for extra in self.extra_resources:
@@ -46,7 +48,7 @@ class ModuleToml:
                 raise ToolkitFileExistsError(f"Extra resource {extra} must be a relative path")
 
     @classmethod
-    def load(cls, data: dict[str, Any] | Path) -> ModuleToml:
+    def load(cls, data: dict[str, Any] | Path) -> Self:
         if isinstance(data, Path):
             return cls.load(toml.loads(data.read_text(encoding="utf-8")))
 
@@ -63,21 +65,21 @@ class ModuleToml:
         if "extra_resources" in data and isinstance(data["extra_resources"], list):
             extra_resources = [Path(item["location"]) for item in data["extra_resources"] if "location" in item]
 
-        tags: frozenset[str] = frozenset()
-        if "packages" in data:
-            if "tags" in data["packages"]:
-                tags = frozenset(data["packages"]["tags"])
         title: str | None = None
+        id: str | None = None
         is_selected_by_default: bool = False
         if "module" in data:
             title = data["module"].get("title")
+            id = data["module"].get("id")
             is_selected_by_default = data["module"].get("is_selected_by_default", False)
+            package_id = data["module"].get("package_id")
 
         return cls(
             title=title,
-            tags=tags,
+            id=id,
             dependencies=dependencies,
             is_selected_by_default=is_selected_by_default,
             data=example_data,
             extra_resources=extra_resources,
+            package_id=package_id,
         )

cognite_toolkit/_cdf_tk/data_classes/_packages.py

@@ -1,18 +1,21 @@
-from __future__ import annotations
-
 import sys
 from collections.abc import ItemsView, Iterable, Iterator, KeysView, Mapping, MutableMapping, ValuesView
 from dataclasses import dataclass, field
 from pathlib import Path
 
-from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitValueError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError
+from cognite_toolkit._cdf_tk.tk_warnings.base import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.tk_warnings.other import LowSeverityWarning
 
 from ._module_directories import ModuleDirectories, ModuleLocation
 
 if sys.version_info >= (3, 11):
+    from typing import Self
+
     import toml
 else:
     import tomli as toml
+    from typing_extensions import Self
 
 
 @dataclass
@@ -22,12 +25,14 @@ class Package:
         name: the unique identifier of the package.
         title: The display name of the package.
         description: A description of the package.
+        id: An optional identifier for package statistics.
         modules: The modules that are part of the package.
     """
 
     name: str
     title: str
     description: str | None = None
+    id: str | None = None
     can_cherry_pick: bool = True
     modules: list[ModuleLocation] = field(default_factory=list)
 
@@ -37,17 +42,24 @@ class Package:
         return {module.name for module in self.modules}
 
     @classmethod
-    def load(cls, name: str, package_definition: dict) -> Package:
+    def load(cls, name: str, package_definition: dict) -> Self:
         return cls(
             name=name,
             title=package_definition["title"],
             description=package_definition.get("description"),
+            id=package_definition.get("id"),
             can_cherry_pick=package_definition.get("canCherryPick", True),
         )
 
 
 class Packages(dict, MutableMapping[str, Package]):
-    def __init__(self, packages: Iterable[Package] | Mapping[str, Package] | None = None) -> None:
+    warnings: WarningList[ToolkitWarning] | None = None
+
+    def __init__(
+        self,
+        packages: Iterable[Package] | Mapping[str, Package] | None = None,
+        warnings: WarningList[ToolkitWarning] | None = None,
+    ) -> None:
         if packages is None:
             super().__init__()
         elif isinstance(packages, Mapping):
@@ -55,40 +67,52 @@ class Packages(dict, MutableMapping[str, Package]):
         else:
             super().__init__({p.name: p for p in packages})
 
+        if warnings:
+            self.warnings = warnings
+
     @classmethod
     def load(
         cls,
-        root_module_dir: Path,  # todo: relative to org dir
-    ) -> Packages:
+        root_module_dir: Path,
+    ) -> Self:
         """Loads the packages in the source directory.
 
         Args:
             root_module_dir: The module directories to load the packages from.
         """
 
-        package_definition_path = root_module_dir / "package.toml"
-        if not package_definition_path.exists():
+        package_definition_path = next(root_module_dir.rglob("packages.toml"), None)
+        if not package_definition_path or not package_definition_path.exists():
             raise ToolkitFileNotFoundError(f"Package manifest toml not found at {package_definition_path}")
-        package_definitions = toml.loads(package_definition_path.read_text(encoding="utf-8"))["packages"]
 
-        collected: dict[str, Package] = {
-            package_name: Package.load(package_name, package_definition)
-            for package_name, package_definition in package_definitions.items()
-            if isinstance(package_definition, dict)
-        }
+        library_definition = toml.loads(package_definition_path.read_text(encoding="utf-8"))
+        package_definitions = library_definition.get("packages", {})
 
+        # Load all available modules
         module_directories = ModuleDirectories.load(root_module_dir)
-        for module in module_directories:
-            if module.definition is None:
-                continue
 
-            for tag in module.definition.tags:
-                if tag in collected:
-                    collected[tag].modules.append(module)
-                else:
-                    raise ToolkitValueError(f"Module {module.name} has an unknown tag {tag}")
+        # Create lookup dictionaries for efficient module discovery
+        module_by_relative_path = {module.relative_path: module for module in module_directories}
+
+        packages_with_modules: dict[str, Package] = {}
+
+        warnings = WarningList[ToolkitWarning]()
+
+        for package_name, package_definition in package_definitions.items():
+            packages_with_modules[package_name] = Package.load(package_name, package_definition)
+            if modules := package_definition.get("modules"):
+                if isinstance(modules, list) and modules:
+                    for module_path in modules:
+                        if (module := module_by_relative_path.get(Path(module_path))) is None:
+                            warnings.append(
+                                LowSeverityWarning(
+                                    f"Unable to load module '{module_path}'. The path may be wrong or the module may require an alpha flag that is not set."
+                                )
+                            )
+                            continue
+                        packages_with_modules[package_name].modules.append(module)
 
-        return cls(collected)
+        return cls(packages_with_modules, warnings)
 
     # The methods are overloads to provide type hints for the methods.
     def items(self) -> ItemsView[str, Package]:  # type: ignore[override]

cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py

@@ -0,0 +1,43 @@
+"""Data class for command tracking information."""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class CommandTrackingInfo(BaseModel):
+    """Structured tracking information for CLI commands.
+
+    This model provides type-safe tracking information that can be collected
+    during command execution and sent to Mixpanel for analytics.
+
+    Attributes:
+        project: The CDF project name.
+        cluster: The CDF cluster name.
+        module_ids: List of module IDs that were deployed or built.
+        package_ids: List of package IDs that were deployed or built.
+        installed_module_ids: List of module IDs that were installed.
+        installed_package_ids: List of package IDs that were installed.
+        downloaded_library_ids: List of library IDs that were downloaded.
+        downloaded_package_ids: List of package IDs that were downloaded.
+        downloaded_module_ids: List of module IDs that were downloaded.
+    """
+
+    project: str | None = Field(default=None)
+    cluster: str | None = Field(default=None)
+    module_ids: set[str] = Field(default_factory=set, alias="moduleIds")
+    package_ids: set[str] = Field(default_factory=set, alias="packageIds")
+    installed_module_ids: set[str] = Field(default_factory=set, alias="installedModuleIds")
+    installed_package_ids: set[str] = Field(default_factory=set, alias="installedPackageIds")
+    downloaded_library_ids: set[str] = Field(default_factory=set, alias="downloadedLibraryIds")
+    downloaded_package_ids: set[str] = Field(default_factory=set, alias="downloadedPackageIds")
+    downloaded_module_ids: set[str] = Field(default_factory=set, alias="downloadedModuleIds")
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert the tracking info to a dictionary for Mixpanel.
+
+        Returns:
+            A dictionary with camelCase keys matching Mixpanel's expected format.
+            Default values are excluded.
+        """
+        return self.model_dump(mode="json", by_alias=True, exclude_defaults=True)

cognite_toolkit/_cdf_tk/exceptions.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import sys
 from collections.abc import Mapping
 from graphlib import CycleError
@@ -181,7 +179,10 @@ class UploadFileError(ToolkitError):
     pass
 
 
-class ResourceRetrievalError(ToolkitError): ...
+class ResourceRetrievalError(ToolkitError):
+    def __init__(self, message: str, resources: list[str] | None = None) -> None:
+        super().__init__(message)
+        self.resources = resources
 
 
 class ResourceCreationError(ToolkitError):
@@ -221,3 +222,31 @@ class ToolkitMissingDependencyError(ToolkitError):
 
     def __init__(self, message: str) -> None:
         super().__init__(message)
+
+
+class ToolkitThrottledError(ToolkitError):
+    """Error raised when a toolkit operation is throttled to respect frequency limits."""
+
+    def __init__(self, message: str, wait_time_seconds: float) -> None:
+        super().__init__(message)
+        self.wait_time_seconds = wait_time_seconds
+
+    def __str__(self) -> str:
+        return f"ToolkitThrottledError: {self.args[0]}"
+
+
+class ToolkitNotImplementedError(NotImplementedError, ToolkitError):
+    """Error raised when a feature is not implemented."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"ToolkitNotImplementedError: {self.message}"
+
+
+class ToolkitRuntimeError(RuntimeError, ToolkitError):
+    """General runtime error for toolkit operations."""
+
+    ...

cognite_toolkit/_cdf_tk/feature_flags.py

@@ -1,57 +1,66 @@
-from __future__ import annotations
-
+from dataclasses import dataclass
 from enum import Enum
 from functools import lru_cache
-from typing import Any, ClassVar
 
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.constants import clean_name
 
 
+@dataclass(frozen=True)
+class FlagMetadata:
+    visible: bool
+    description: str
+
+
 class Flags(Enum):
-    # Ruff requires annotations while mypy requires no annotations
-    INTERNAL: ClassVar[dict[str, Any]] = {"visible": False, "description": "Does nothing"}  # type: ignore[misc]
-    IMPORT_CMD: ClassVar[dict[str, Any]] = {"visible": True, "description": "Enables the import sub application"}  # type: ignore[misc]
-    GRAPHQL: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables the support for deploying data models as GraphQL schemas",
-    }
-    MODULE_REPEAT: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables the support for repeating modules in the config file",
-    }
-    POPULATE: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables support for the populate command",
-    }
-    AGENTS: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables support for Atlas Agents and Agent Tools",
-    }
-    DUMP_EXTENDED: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables support for the dumping Location Filters.",
-    }
-    PROFILE: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables support for the profile command",
-    }
-    INFIELD: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables support for Infield configs",
-    }
-    DUMP_DATA: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Splits the dump command in dump data and dump config",
-    }
-    EXIT_ON_WARNING: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables the exit on warning feature during the build command",
-    }
-    MIGRATE: ClassVar[dict[str, Any]] = {  # type: ignore[misc]
-        "visible": True,
-        "description": "Enables the migrate command",
-    }
+    INTERNAL = FlagMetadata(
+        visible=False,
+        description="Does nothing",
+    )
+    IMPORT_CMD = FlagMetadata(
+        visible=True,
+        description="Enables the import sub application",
+    )
+    GRAPHQL = FlagMetadata(
+        visible=False,
+        description="Enables the support for deploying data models as GraphQL schemas",
+    )
+    MODULE_REPEAT = FlagMetadata(
+        visible=True,
+        description="Enables the support for repeating modules in the config file",
+    )
+    PROFILE = FlagMetadata(
+        visible=True,
+        description="Enables support for the profile command",
+    )
+    INFIELD = FlagMetadata(
+        visible=True,
+        description="Enables support for Infield configs",
+    )
+    MIGRATE = FlagMetadata(
+        visible=True,
+        description="Enables the migrate command",
+    )
+    STREAMS = FlagMetadata(
+        visible=True,
+        description="Enables the support for the streams resources",
+    )
+    v08 = FlagMetadata(
+        visible=False,
+        description="Enables features planned for Cognite Toolkit version 0.8.0",
+    )
+    CREATE = FlagMetadata(
+        visible=True,
+        description="Enables the support for the resources create command under dev plugin",
+    )
+    EXTEND_DOWNLOAD = FlagMetadata(
+        visible=True,
+        description="Enables extended download to support downloading file content and datapoints",
+    )
+    EXTEND_UPLOAD = FlagMetadata(
+        visible=True,
+        description="Enables extended upload to support uploading individual files",
+    )
 
     def is_enabled(self) -> bool:
         return FeatureFlag.is_enabled(self)
@@ -62,3 +71,7 @@ class FeatureFlag:
     @lru_cache(typed=True)
     def is_enabled(flag: Flags) -> bool:
         return CDFToml.load().alpha_flags.get(clean_name(flag.name), False)
+
+    @staticmethod
+    def flush() -> None:
+        FeatureFlag.is_enabled.cache_clear()

cognite_toolkit/_cdf_tk/hints.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import sys
 from abc import abstractmethod
 from pathlib import Path
@@ -11,8 +9,8 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 
 from .constants import COGNITE_MODULES, CUSTOM_MODULES, HINT_LEAD_TEXT, MODULES, ROOT_MODULES, URL
+from .cruds import CRUDS_BY_FOLDER_NAME
 from .exceptions import ToolkitFileNotFoundError, ToolkitNotADirectoryError
-from .loaders import LOADER_BY_FOLDER_NAME
 from .tk_warnings import MediumSeverityWarning
 from .utils import find_directory_with_subdirectories
 
@@ -47,13 +45,15 @@ class Hint:
 class ModuleDefinition(Hint):
     @classmethod
     def _short(cls) -> str:
-        return f"Available resource directories are {sorted(LOADER_BY_FOLDER_NAME)}. {cls.link(URL.configs)} to learn more."
+        return (
+            f"Available resource directories are {sorted(CRUDS_BY_FOLDER_NAME)}. {cls.link(URL.configs)} to learn more."
+        )
 
     @classmethod
     def long(cls, missing_modules: set[str | Path] | None = None, organization_dir: Path | None = None) -> str:  # type: ignore[override]
         lines = [
             "A module is a directory with one or more resource directories in it.",
-            f"Available resource directories are {sorted(LOADER_BY_FOLDER_NAME)}",
+            f"Available resource directories are {sorted(CRUDS_BY_FOLDER_NAME)}",
             f"{cls.link(URL.configs)} to learn more",
         ]
         if missing_modules and organization_dir:

cognite_toolkit/_cdf_tk/plugins.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from dataclasses import dataclass
 from enum import Enum
 
@@ -19,7 +17,8 @@ class Plugin:
 class Plugins(Enum):
     run = Plugin("run", "plugin for Run command to execute Python scripts in CDF")
     dump = Plugin("dump", "plugin for Dump command to retrieve Asset resources from CDF")
-    purge = Plugin("purge", "plugin for Purge command to remove datasets and spaces from CDF")
+    dev = Plugin("dev", "plugin for commands to develop modules in CDF")
+    data = Plugin("data", "plugin for Data command to manage data in CDF")
 
     @staticmethod
     def list() -> dict[str, bool]:

cognite_toolkit/_cdf_tk/protocols.py

@@ -0,0 +1,97 @@
+import sys
+from collections.abc import Collection, Iterator
+from typing import Any, Generic, Protocol, TypeVar
+
+from cognite.client import CogniteClient
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class ResourceRequestProtocol(Protocol):
+    @classmethod
+    def _load(cls, data: dict[str, Any]) -> Self: ...
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]: ...
+
+
+class ResourceResponseProtocol(Protocol):
+    def as_write(self) -> ResourceRequestProtocol: ...
+
+
+T_ResourceRequest = TypeVar("T_ResourceRequest", bound=ResourceRequestProtocol)
+T_ResourceResponse = TypeVar("T_ResourceResponse", bound=ResourceResponseProtocol)
+
+
+class ResourceRequestListProtocol(Protocol, Generic[T_ResourceRequest]):
+    def __init__(self, collection: Collection[T_ResourceRequest]): ...
+
+    def __getitem__(self, index: int) -> T_ResourceRequest: ...
+
+    def __setitem__(self, index: int, value: T_ResourceRequest) -> None: ...
+
+    def __delitem__(self, index: int) -> None: ...
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[T_ResourceRequest]: ...
+
+    def __contains__(self, value: object) -> bool: ...
+
+    def __reversed__(self) -> Iterator[T_ResourceRequest]: ...
+
+    def insert(self, index: int, value: T_ResourceRequest) -> None: ...
+
+    def append(self, value: T_ResourceRequest) -> None: ...
+
+    def extend(self, values: Collection[T_ResourceRequest]) -> None: ...
+
+    def pop(self, index: int = -1) -> T_ResourceRequest: ...
+
+    def remove(self, value: T_ResourceRequest) -> None: ...
+
+    def clear(self) -> None: ...
+
+    @classmethod
+    def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> Self: ...
+
+
+class ResourceResponseListProtocol(Protocol, Generic[T_ResourceResponse]):
+    def __init__(self, collection: Collection[T_ResourceResponse]): ...
+
+    def __getitem__(self, index: int) -> T_ResourceResponse: ...
+
+    def __setitem__(self, index: int, value: T_ResourceResponse) -> None: ...
+
+    def __delitem__(self, index: int) -> None: ...
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[T_ResourceResponse]: ...
+
+    def __contains__(self, value: object) -> bool: ...
+
+    def __reversed__(self) -> Iterator[T_ResourceResponse]: ...
+
+    def insert(self, index: int, value: T_ResourceResponse) -> None: ...
+
+    def append(self, value: T_ResourceResponse) -> None: ...
+
+    def extend(self, values: Collection[T_ResourceResponse]) -> None: ...
+
+    def pop(self, index: int = -1) -> T_ResourceResponse: ...
+
+    def remove(self, value: T_ResourceResponse) -> None: ...
+
+    def clear(self) -> None: ...
+
+    def as_write(self) -> ResourceRequestListProtocol: ...
+
+    @classmethod
+    def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> Self: ...
+
+
+T_ResourceRequestList = TypeVar("T_ResourceRequestList", bound=ResourceRequestListProtocol)
+T_ResourceResponseList = TypeVar("T_ResourceResponseList", bound=ResourceResponseListProtocol)