cognite-toolkit 0.7.46__py3-none-any.whl → 0.7.48__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_modules.py

@@ -0,0 +1,27 @@
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from cognite_toolkit._cdf_tk.data_classes._module_toml import ModuleToml
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Module(BaseModel):
+    model_config = ConfigDict(
+        frozen=True,
+        validate_assignment=True,
+        arbitrary_types_allowed=True,
+    )
+
+    path: Path
+    definition: ModuleToml | None = None
+
+    @classmethod
+    def load(cls, path: Path) -> Self:
+        definition = ModuleToml.load(path / ModuleToml.filename) if (path / ModuleToml.filename).exists() else None
+        return cls(path=path, definition=definition)

cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_resource.py

@@ -0,0 +1,22 @@
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Resource(BaseModel):
+    model_config = ConfigDict(
+        frozen=True,
+        validate_assignment=True,
+    )
+
+    path: Path
+
+    @classmethod
+    def load(cls, path: Path) -> Self:
+        return cls(path=path)

cognite_toolkit/_cdf_tk/cruds/__init__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import itertools
+from collections import defaultdict
 from typing import Literal, TypeAlias
 
 from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags
@@ -90,24 +91,28 @@ if not FeatureFlag.is_enabled(Flags.STREAMS):
 if not FeatureFlag.is_enabled(Flags.SIMULATORS):
     _EXCLUDED_CRUDS.add(SimulatorModelCRUD)
 
-CRUDS_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
+CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA: defaultdict[str, list[type[Loader]]] = defaultdict(list)
+CRUDS_BY_FOLDER_NAME: defaultdict[str, list[type[Loader]]] = defaultdict(list)
 for _loader in itertools.chain(
     ResourceCRUD.__subclasses__(),
     ResourceContainerCRUD.__subclasses__(),
     DataCRUD.__subclasses__(),
     GroupCRUD.__subclasses__(),
 ):
-    if _loader in [ResourceCRUD, ResourceContainerCRUD, DataCRUD, GroupCRUD] or _loader in _EXCLUDED_CRUDS:
+    if _loader in [ResourceCRUD, ResourceContainerCRUD, DataCRUD, GroupCRUD]:
         # Skipping base classes
         continue
-    if _loader.folder_name not in CRUDS_BY_FOLDER_NAME:  # type: ignore[attr-defined]
-        CRUDS_BY_FOLDER_NAME[_loader.folder_name] = []  # type: ignore[attr-defined]
     # MyPy bug: https://github.com/python/mypy/issues/4717
-    CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
+    CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
+
+    if _loader not in _EXCLUDED_CRUDS:
+        CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
 del _loader  # cleanup module namespace
 
+
 # For backwards compatibility
 CRUDS_BY_FOLDER_NAME["data_models"] = CRUDS_BY_FOLDER_NAME["data_modeling"]  # Todo: Remove in v1.0
+CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA["data_models"] = CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA["data_modeling"]
 RESOURCE_CRUD_BY_FOLDER_NAME = {
     folder_name: cruds
     for folder_name, loaders in CRUDS_BY_FOLDER_NAME.items()
@@ -166,6 +171,7 @@ __all__ = [
     "RESOURCE_CRUD_CONTAINER_LIST",
     "RESOURCE_CRUD_LIST",
     "RESOURCE_DATA_CRUD_LIST",
+    "_EXCLUDED_CRUDS",
     "AgentCRUD",
     "AssetCRUD",
     "CogniteFileCRUD",

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -4,11 +4,10 @@ from typing import Any, final
 from cognite.client.data_classes.capabilities import Capability, StreamsAcl
 from cognite.client.utils.useful_types import SequenceNotStr
 
-from cognite_toolkit._cdf_tk.client.http_client import ToolkitAPIError
+from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId
 from cognite_toolkit._cdf_tk.client.resource_classes.streams import (
     StreamRequest,
     StreamResponse,
-    StreamResponseList,
 )
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.resource_classes import StreamYAML
@@ -17,7 +16,7 @@ from .datamodel import ContainerCRUD
 
 
 @final
-class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
+class StreamCRUD(ResourceCRUD[ExternalId, StreamRequest, StreamResponse]):
     folder_name = "streams"
     resource_cls = StreamResponse
     resource_write_cls = StreamRequest
@@ -32,14 +31,14 @@ class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
         return "streams"
 
     @classmethod
-    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> str:
+    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> ExternalId:
         if isinstance(item, dict):
-            return item["externalId"]
-        return item.external_id
+            return ExternalId(external_id=item["externalId"])
+        return ExternalId(external_id=item.external_id)
 
     @classmethod
-    def dump_id(cls, id: str) -> dict[str, Any]:
-        return {"externalId": id}
+    def dump_id(cls, id: ExternalId) -> dict[str, Any]:
+        return id.dump()
 
     @classmethod
     def get_required_capability(
@@ -55,30 +54,15 @@ class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
         )
         return StreamsAcl(actions, StreamsAcl.Scope.All())
 
-    def create(self, items: Sequence[StreamRequest]) -> StreamResponseList:
-        created = self.client.streams.create(list(items))
-        return StreamResponseList(created)
+    def create(self, items: Sequence[StreamRequest]) -> list[StreamResponse]:
+        return self.client.streams.create(items)
 
-    def retrieve(self, ids: SequenceNotStr[str]) -> StreamResponseList:
-        retrieved: list[StreamResponse] = []
-        for _id in ids:
-            try:
-                _resp = self.client.streams.retrieve(_id)
-            except ToolkitAPIError:
-                continue
-            if _resp is not None:
-                retrieved.append(_resp)
-        return StreamResponseList(retrieved)
+    def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[StreamResponse]:
+        return self.client.streams.retrieve(list(ids), ignore_unknown_ids=True)
 
-    def delete(self, ids: SequenceNotStr[str]) -> int:
-        count = 0
-        for _id in ids:
-            try:
-                self.client.streams.delete(_id)
-            except ToolkitAPIError:
-                continue
-            count += 1
-        return count
+    def delete(self, ids: SequenceNotStr[ExternalId]) -> int:
+        self.client.streams.delete(list(ids), ignore_unknown_ids=True)
+        return len(ids)
 
     def _iterate(
         self,

cognite_toolkit/_cdf_tk/data_classes/__init__.py

@@ -29,6 +29,7 @@ from ._deploy_results import (
     ResourceDeployResult,
     UploadDeployResult,
 )
+from ._issues import Issue, IssueList
 from ._module_directories import ModuleDirectories, ModuleLocation
 from ._module_resources import ModuleResources
 from ._packages import Package, Packages
@@ -56,6 +57,8 @@ __all__ = [
     "DeployResults",
     "Environment",
     "InitConfigYAML",
+    "Issue",
+    "IssueList",
     "ModuleDirectories",
     "ModuleLocation",
     "ModuleResources",

cognite_toolkit/_cdf_tk/data_classes/_issues.py

@@ -0,0 +1,36 @@
+import sys
+from collections import UserList
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from pydantic import BaseModel, Field
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+
+MODULE_ISSUE_CODE = "MOD"
+
+
+class Issue(BaseModel):
+    """Base class for all issues"""
+
+    code: str
+    message: str | None = Field(default=None)
+
+
+# temporary adapter to manage existing warnings
+class IssueList(UserList[Issue]):
+    """List of build issues."""
+
+    @classmethod
+    def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self:
+        """Create a IssueList from a WarningList."""
+        return cls([Issue(code="WARN", message=warning.get_message()) for warning in warning_list])
+
+
+class ModuleLoadingIssue(Issue):
+    """Issue with the loading of a module folder."""
+
+    code: str = "MOD_001"

cognite_toolkit/_cdf_tk/data_classes/_module_directories.py

@@ -8,7 +8,8 @@ from pathlib import Path
 from typing import Any, SupportsIndex, overload
 
 from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN
-from cognite_toolkit._cdf_tk.utils import calculate_directory_hash, iterate_modules, resource_folder_from_path
+from cognite_toolkit._cdf_tk.utils.hashing import calculate_directory_hash
+from cognite_toolkit._cdf_tk.utils.modules import iterate_modules, resource_folder_from_path
 
 from ._module_toml import ModuleToml
 

cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -11,6 +11,7 @@ from cognite_toolkit._cdf_tk.utils.collection import chunker
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
+from .logger import DataLogger, NoOpLogger
 from .selectors import DataSelector
 
 
@@ -76,6 +77,7 @@ class StorageIO(ABC, Generic[T_Selector, T_ResourceResponse]):
 
     def __init__(self, client: ToolkitClient) -> None:
         self.client = client
+        self.logger: DataLogger = NoOpLogger()
 
     @abstractmethod
     def as_id(self, item: T_ResourceResponse) -> str:

cognite_toolkit/_cdf_tk/storageio/logger.py

@@ -0,0 +1,163 @@
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from collections.abc import Sequence
+from threading import Lock
+from typing import Literal, TypeAlias
+
+from pydantic import BaseModel
+from pydantic.alias_generators import to_camel
+
+from cognite_toolkit._cdf_tk.utils.fileio import NDJsonWriter
+
+
+class LogEntry(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
+    """Represents a log entry for tracking storage I/O operations."""
+
+    id: str
+    message: str
+
+
+OperationStatus: TypeAlias = Literal["success", "failure", "unchanged", "pending"]
+
+
+class ItemTracker:
+    """Tracks issues accumulated for a single item during pipeline processing."""
+
+    def __init__(self, item_id: str) -> None:
+        self.item_id = item_id
+        self.issues: list[str] = []
+
+    def add_issue(self, issue: str) -> None:
+        """Add an issue encountered during processing."""
+        self.issues.append(issue)
+
+
+class OperationTracker(ABC):
+    """Abstract base class for operation trackers."""
+
+    @abstractmethod
+    def add_issue(self, item_id: str, issue: str) -> None:
+        """Add an issue to an item."""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def finalize_item(self, item_id: str | list[str], status: OperationStatus) -> None:
+        """Finalize an item with its final status."""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_status_counts(self) -> dict[OperationStatus, int]:
+        """Get counts per final status."""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_issue_counts(self, status: OperationStatus) -> dict[str, int]:
+        """Get issue counts, optionally filtered by status."""
+        raise NotImplementedError()
+
+
+class NoOpTracker(OperationTracker):
+    """A no-op tracker that does nothing."""
+
+    def add_issue(self, item_id: str, issue: str) -> None:
+        """No-op: Discard the issue."""
+        pass
+
+    def finalize_item(self, item_id: str | list[str], status: OperationStatus) -> None:
+        """No-op: Do nothing."""
+        pass
+
+    def get_status_counts(self) -> dict[OperationStatus, int]:
+        """Return empty status counts."""
+        return {}
+
+    def get_issue_counts(self, status: OperationStatus) -> dict[str, int]:
+        """Return empty issue counts."""
+        return {}
+
+
+class MemoryOperationTracker(OperationTracker):
+    """Tracks the overall operation progress and issues across multiple items.
+
+    Tracks counts of final statuses and issues per status.
+    """
+
+    def __init__(self) -> None:
+        self._lock = Lock()
+        self._active_items: dict[str, ItemTracker] = {}
+        self._status_counts: dict[OperationStatus, int] = defaultdict(int)
+        self._issue_counts: dict[OperationStatus, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+
+    def add_issue(self, item_id: str, issue: str) -> None:
+        """Add an issue to an item, creating tracker if needed."""
+        with self._lock:
+            if item_id not in self._active_items:
+                self._active_items[item_id] = ItemTracker(item_id)
+            self._active_items[item_id].add_issue(issue)
+
+    def finalize_item(self, item_id: str | list[str], status: OperationStatus) -> None:
+        """Finalize an item with its final status.
+
+        Args:
+            item_id: The item's identifier.
+            status: Final status (success, failure, unchanged).
+        """
+        with self._lock:
+            if isinstance(item_id, list):
+                for iid in item_id:
+                    self._finalize_item_unlocked(iid, status)
+            else:
+                self._finalize_item_unlocked(item_id, status)
+
+    def _finalize_item_unlocked(self, item_id: str, status: OperationStatus) -> None:
+        """Internal method to finalize an item without acquiring the lock."""
+        tracker = self._active_items.pop(item_id, None)
+        self._status_counts[status] += 1
+        if tracker is not None:
+            for issue in tracker.issues:
+                self._issue_counts[status][issue] += 1
+
+    def get_status_counts(self) -> dict[OperationStatus, int]:
+        """Get counts per final status."""
+        with self._lock:
+            return dict(self._status_counts)
+
+    def get_issue_counts(self, status: OperationStatus) -> dict[str, int]:
+        """Get issue counts, optionally filtered by status."""
+        with self._lock:
+            return dict(self._issue_counts.get(status, {}))
+
+
+class DataLogger(ABC):
+    """Abstract base class for data loggers that track operations and log entries."""
+
+    tracker: OperationTracker
+
+    @abstractmethod
+    def log(self, entry: LogEntry | Sequence[LogEntry]) -> None:
+        """Log a detailed entry."""
+        raise NotImplementedError()
+
+
+class NoOpLogger(DataLogger):
+    """A no-op logger that discards all log entries and does no tracking."""
+
+    def __init__(self) -> None:
+        self.tracker = NoOpTracker()
+
+    def log(self, entry: LogEntry | Sequence[LogEntry]) -> None:
+        """Discard the log entry (no-op)."""
+        pass
+
+
+class FileDataLogger(DataLogger):
+    """Composes aggregation tracking with detailed file logging."""
+
+    def __init__(self, writer: NDJsonWriter) -> None:
+        self.tracker = MemoryOperationTracker()
+        self._writer = writer
+
+    def log(self, entry: LogEntry | Sequence[LogEntry]) -> None:
+        """Log a detailed entry to the file."""
+        entries = entry if isinstance(entry, Sequence) else [entry]
+        self._writer.write_chunks([e.model_dump(by_alias=True) for e in entries])

cognite_toolkit/_cdf_tk/utils/__init__.py

@@ -19,7 +19,13 @@ from .hashing import (
     calculate_hash,
     calculate_secure_hash,
 )
-from .modules import find_directory_with_subdirectories, iterate_modules, module_from_path, resource_folder_from_path
+from .modules import (
+    find_directory_with_subdirectories,
+    iterate_modules,
+    module_from_path,
+    module_path_display_name,
+    resource_folder_from_path,
+)
 from .sentry_utils import sentry_exception_filter
 
 __all__ = [
@@ -37,6 +43,7 @@ __all__ = [
     "iterate_modules",
     "load_yaml_inject_variables",
     "module_from_path",
+    "module_path_display_name",
     "quote_int_value_by_key_in_yaml",
     "read_yaml_content",
     "read_yaml_file",

cognite_toolkit/_cdf_tk/utils/interactive_select.py

@@ -852,7 +852,9 @@ class ThreeDInteractiveSelect:
             ],
         ).ask()
 
-        models = self.client.tool.three_d.models.list(published=published, include_revision_info=True, limit=None)
+        models = self.client.tool.three_d.models_classic.list(
+            published=published, include_revision_info=True, limit=None
+        )
         if model_type == "classic":
             models = [model for model in models if model.space is None]
         else:

cognite_toolkit/_cdf_tk/utils/modules.py

@@ -12,6 +12,7 @@ from cognite_toolkit._cdf_tk.constants import (
 def iterate_modules(root_dir: Path) -> Iterator[tuple[Path, list[Path]]]:
     """Iterate over all modules in the project and yield the module directory and all files in the module.
 
+
     Args:
         root_dir (Path): The root directory of the project
 
@@ -110,6 +111,12 @@ def is_module_path(path: Path) -> bool:
     return any(sub_folder.name in CRUDS_BY_FOLDER_NAME for sub_folder in path.iterdir() if sub_folder.is_dir())
 
 
+def module_path_display_name(org_path: Path, module_path: Path) -> str:
+    """Returns the module directory path relative to the organization path as a posix string."""
+    rel = module_path.relative_to(org_path)
+    return rel.as_posix()
+
+
 def find_directory_with_subdirectories(
     directory_name: str | None, root_directory: Path
 ) -> tuple[Path | None, list[str]]:

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.46
+      image: cognite/toolkit:0.7.48
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.46
+      image: cognite/toolkit:0.7.48
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.46"
+version = "0.7.48"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.46"
+__version__ = "0.7.48"

{cognite_toolkit-0.7.46.dist-info → cognite_toolkit-0.7.48.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.46
+Version: 0.7.48
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>