cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py

@@ -0,0 +1,187 @@
+import gzip
+import sys
+from abc import ABC, abstractmethod
+from collections.abc import Hashable
+from typing import Any, Literal
+
+import httpx
+from cognite.client import global_config
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, model_validator
+from pydantic.alias_generators import to_camel
+
+from cognite_toolkit._cdf_tk.utils.http_client._tracker import ItemsRequestTracker
+from cognite_toolkit._cdf_tk.utils.useful_types import PrimitiveType
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class HTTPResult2(BaseModel): ...
+
+
+class FailedRequest2(HTTPResult2):
+    error: str
+
+
+class SuccessResponse2(HTTPResult2):
+    status_code: int
+    body: str
+    content: bytes
+
+
+class ErrorDetails2(BaseModel):
+    """This is the expected structure of error details in the CDF API"""
+
+    code: int
+    message: str
+    missing: list[JsonValue] | None = None
+    duplicated: list[JsonValue] | None = None
+    is_auto_retryable: bool | None = None
+
+    @classmethod
+    def from_response(cls, response: httpx.Response) -> "ErrorDetails2":
+        """Populate the error details from a httpx response."""
+        try:
+            res = TypeAdapter(dict[Literal["error"], ErrorDetails2]).validate_json(response.text)
+        except ValueError:
+            return cls(code=response.status_code, message=response.text)
+        return res["error"]
+
+
+class FailedResponse2(HTTPResult2):
+    status_code: int
+    body: str
+    error: ErrorDetails2
+
+
+class BaseRequestMessage(BaseModel, ABC):
+    endpoint_url: str
+    method: Literal["GET", "POST", "PATCH", "DELETE", "PUT"]
+    connect_attempt: int = 0
+    read_attempt: int = 0
+    status_attempt: int = 0
+    api_version: str | None = None
+    content_type: str = "application/json"
+    accept: str = "application/json"
+
+    parameters: dict[str, PrimitiveType] | None = None
+
+    @property
+    def total_attempts(self) -> int:
+        return self.connect_attempt + self.read_attempt + self.status_attempt
+
+    @property
+    @abstractmethod
+    def content(self) -> str | bytes | None: ...
+
+
+class RequestMessage2(BaseRequestMessage):
+    data_content: bytes | None = None
+    body_content: dict[str, JsonValue] | None = None
+
+    @model_validator(mode="before")
+    def check_data_or_body(cls, values: dict[str, Any]) -> dict[str, Any]:
+        if values.get("data_content") is not None and values.get("body_content") is not None:
+            raise ValueError("Only one of data_content or body_content can be set.")
+        return values
+
+    @property
+    def content(self) -> str | bytes | None:
+        data: str | bytes | None = None
+        if self.data_content is not None:
+            data = self.data_content
+            if not global_config.disable_gzip:
+                data = gzip.compress(data)
+        elif self.body_content is not None:
+            # We serialize using pydantic instead of json.dumps. This is because pydantic is faster
+            # and handles more complex types such as datetime, float('nan'), etc.
+            data = _BODY_SERIALIZER.dump_json(self.body_content)
+            if not global_config.disable_gzip and isinstance(data, str):
+                data = gzip.compress(data.encode("utf-8"))
+        return data
+
+
+_BODY_SERIALIZER = TypeAdapter(dict[str, JsonValue])
+
+
+class ItemsResultMessage2(BaseModel):
+    ids: list[Hashable]
+
+
+class ItemsFailedRequest2(ItemsResultMessage2):
+    error_message: str
+
+
+class ItemsSuccessResponse2(ItemsResultMessage2):
+    status_code: int
+    body: str
+    content: bytes
+
+
+class ItemsFailedResponse2(ItemsResultMessage2):
+    status_code: int
+    error: ErrorDetails2
+    body: str
+
+
+class BaseModelObject(BaseModel):
+    """Base class for all object. This includes resources and nested objects."""
+
+    # We allow extra fields to support forward compatibility.
+    model_config = ConfigDict(alias_generator=to_camel, extra="allow")
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]:
+        """Dump the resource to a dictionary.
+
+        This is the default serialization method for request resources.
+        """
+        return self.model_dump(mode="json", by_alias=camel_case, exclude_unset=True)
+
+    @classmethod
+    def _load(cls, resource: dict[str, Any]) -> "Self":
+        """Load method to match CogniteResource signature."""
+        return cls.model_validate(resource)
+
+
+class RequestResource(BaseModelObject, ABC):
+    @abstractmethod
+    def as_id(self) -> Hashable: ...
+
+
+def _set_default_tracker(data: dict[str, Any]) -> ItemsRequestTracker:
+    if "tracker" not in data or data["tracker"] is None:
+        return ItemsRequestTracker(data.get("max_failures_before_abort", 50))
+    return data["tracker"]
+
+
+class ItemsRequest2(BaseRequestMessage):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    items: list[RequestResource]
+    extra_body_fields: dict[str, JsonValue] | None = None
+    max_failures_before_abort: int = 50
+    tracker: ItemsRequestTracker = Field(init=False, default_factory=_set_default_tracker, exclude=True)
+
+    @property
+    def content(self) -> str | bytes | None:
+        body: dict[str, JsonValue] = {"items": [item.dump() for item in self.items]}
+        if self.extra_body_fields:
+            body.update(self.extra_body_fields)
+        res = _BODY_SERIALIZER.dump_json(body)
+        if not global_config.disable_gzip and isinstance(res, str):
+            return gzip.compress(res.encode("utf-8"))
+        return res
+
+    def split(self, status_attempts: int) -> list["ItemsRequest2"]:
+        """Split the request into multiple requests with a single item each."""
+        mid = len(self.items) // 2
+        if mid == 0:
+            return [self]
+        self.tracker.register_failure()
+        messages: list[ItemsRequest2] = []
+        for part in (self.items[:mid], self.items[mid:]):
+            new_request = self.model_copy(update={"items": part, "status_attempt": status_attempts})
+            new_request.tracker = self.tracker
+            messages.append(new_request)
+        return messages

cognite_toolkit/_cdf_tk/utils/interactive_select.py

@@ -14,9 +14,6 @@ from cognite.client.data_classes import (
     filters,
 )
 from cognite.client.data_classes.aggregations import Count
-from cognite.client.data_classes.capabilities import (
-    UserProfilesAcl,
-)
 from cognite.client.data_classes.data_modeling import ContainerId, NodeList, Space, SpaceList, View, ViewId, ViewList
 from cognite.client.data_classes.data_modeling.statistics import SpaceStatistics
 from cognite.client.utils import ms_to_datetime
@@ -28,7 +25,7 @@ from cognite_toolkit._cdf_tk.client.data_classes.canvas import Canvas
 from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartList, Visibility
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
 from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
-from cognite_toolkit._cdf_tk.exceptions import AuthorizationError, ToolkitMissingResourceError, ToolkitValueError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError, ToolkitValueError
 
 from . import humanize_collection
 from .aggregators import (
@@ -309,9 +306,14 @@ class InteractiveCanvasSelect:
     opening_choices: ClassVar[list[questionary.Choice]] = [
         questionary.Choice(title="All public Canvases", value=CanvasFilter(visibility="public", select_all=True)),
         questionary.Choice(title="Selected public Canvases", value=CanvasFilter(visibility="public", select_all=False)),
-        questionary.Choice(title="All by given user", value=CanvasFilter(created_by="user", select_all=True)),
-        questionary.Choice(title="Selected by given user", value=CanvasFilter(created_by="user", select_all=False)),
-        questionary.Choice(title="All Canvases", value=CanvasFilter(visibility=None, select_all=True)),
+        questionary.Choice(
+            title="All public Canvases by given user",
+            value=CanvasFilter(created_by="user", select_all=True, visibility="public"),
+        ),
+        questionary.Choice(
+            title="Selected public Canvases by given user",
+            value=CanvasFilter(created_by="user", select_all=False, visibility="public"),
+        ),
     ]
 
     def __init__(self, client: ToolkitClient) -> None:
@@ -406,13 +408,6 @@ class InteractiveChartSelect:
         return user_response
 
     def _select_external_ids(self, select_filter: ChartFilter) -> list[str]:
-        if not self.client.iam.verify_capabilities(
-            UserProfilesAcl([UserProfilesAcl.Action.Read], scope=UserProfilesAcl.Scope.All())
-        ):
-            raise AuthorizationError(
-                "The current user does not have permission to list user profiles, "
-                "which is required to select Charts owned by a specific user."
-            )
         available_charts = self.client.charts.list(visibility=(select_filter.visibility or "PUBLIC"))
         if select_filter.select_all and select_filter.owned_by is None:
             return [chart.external_id for chart in available_charts]

cognite_toolkit/_cdf_tk/utils/sql_parser.py

@@ -1,12 +1,11 @@
 import importlib.util
+from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
 from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingDependencyError
 
 if TYPE_CHECKING:
-    from sqlparse.sql import Identifier
-    from sqlparse.tokens import Token
-from dataclasses import dataclass
+    from sqlparse.sql import Identifier, Token
 
 
 @dataclass(frozen=True)

cognite_toolkit/_cdf_tk/utils/useful_types.py

@@ -8,10 +8,13 @@ from cognite.client.data_classes._base import CogniteObject, WriteableCogniteRes
 JsonVal: TypeAlias = None | str | int | float | bool | dict[str, "JsonVal"] | list["JsonVal"]
 
 AssetCentricDestinationType: TypeAlias = Literal["assets", "files", "events", "timeseries", "sequences"]
-AssetCentricType: TypeAlias = Literal["asset", "file", "event", "timeseries", "sequence", "fileAnnotation"]
+AssetCentricType: TypeAlias = Literal["asset", "file", "event", "timeseries", "sequence"]
+AssetCentricTypeExtended: TypeAlias = Literal["asset", "file", "event", "timeseries", "sequence", "annotation"]
 AssetCentricResource: TypeAlias = Asset | FileMetadata | Event | TimeSeries
 AssetCentricResourceExtended: TypeAlias = Asset | FileMetadata | Event | TimeSeries | Annotation
-AssetCentricKind: TypeAlias = Literal["Assets", "Events", "TimeSeries", "FileMetadata", "FileAnnotations"]
+AssetCentricKind: TypeAlias = Literal["Assets", "Events", "TimeSeries", "FileMetadata"]
+AssetCentricKindExtended: TypeAlias = Literal["Assets", "Events", "TimeSeries", "FileMetadata", "Annotations"]
+
 
 DataType: TypeAlias = Literal["string", "integer", "float", "boolean", "json", "date", "timestamp", "epoch"]
 PythonTypes: TypeAlias = str | int | float | bool | datetime | date | dict[str, Any] | list[Any]
@@ -25,3 +28,4 @@ PrimitiveType: TypeAlias = str | int | float | bool
 
 T_WriteCogniteResource = TypeVar("T_WriteCogniteResource", bound=CogniteObject)
 T_AssetCentricResource = TypeVar("T_AssetCentricResource", bound=AssetCentricResource)
+T_AssetCentricResourceExtended = TypeVar("T_AssetCentricResourceExtended", bound=AssetCentricResourceExtended)

cognite_toolkit/_cdf_tk/validation.py

@@ -8,18 +8,29 @@ from cognite.client.utils._text import to_snake_case
 from pydantic import BaseModel, TypeAdapter, ValidationError
 from pydantic_core import ErrorDetails
 
-from cognite_toolkit._cdf_tk.data_classes import BuildVariables
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.constants import DEV_ONLY_MODULES
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, ModuleDirectories
+from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitDuplicatedModuleError,
+    ToolkitEnvError,
+    ToolkitMissingModuleError,
+)
+from cognite_toolkit._cdf_tk.hints import ModuleDefinition
 from cognite_toolkit._cdf_tk.resource_classes import BaseModelResource
 from cognite_toolkit._cdf_tk.tk_warnings import (
     DataSetMissingWarning,
+    MediumSeverityWarning,
     TemplateVariableWarning,
     WarningList,
 )
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.utils import humanize_collection
 
 __all__ = [
     "humanize_validation_error",
     "validate_data_set_is_set",
+    "validate_module_selection",
     "validate_modules_variables",
 ]
 
@@ -210,3 +221,70 @@ def as_json_path(loc: tuple[str | int, ...]) -> str:
 
     suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
     return f"{prefix}{suffix}"
+
+
+def validate_module_selection(
+    modules: ModuleDirectories,
+    config: BuildConfigYAML,
+    packages: dict[str, list[str]],
+    selected_modules: set[str | Path],
+    organization_dir: Path,
+) -> WarningList:
+    """Validates module selection and returns warnings for non-critical issues.
+
+    Critical errors (duplicate modules, missing modules, no modules selected) are still raised
+    as exceptions as they prevent the build from proceeding.
+    """
+    warnings: WarningList = WarningList()
+
+    # Validations: Ambiguous selection.
+    selected_names = {s for s in config.environment.selected if isinstance(s, str)}
+    if duplicate_modules := {
+        module_name: paths
+        for module_name, paths in modules.as_path_by_name().items()
+        if len(paths) > 1 and module_name in selected_names
+    }:
+        # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
+        # Note, if the user uses a path to select a module, this error will not be raised.
+        raise ToolkitDuplicatedModuleError(
+            f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
+        )
+
+    # Package Referenced Modules Exists
+    for package, package_modules in packages.items():
+        if package not in selected_names:
+            # We do not check packages that are not selected.
+            # Typically, the user will delete the modules that are irrelevant for them;
+            # thus we only check the selected packages.
+            continue
+        if missing_packages := set(package_modules) - modules.available_names:
+            raise ToolkitMissingModuleError(
+                f"Package {package} defined in {CDFToml.file_name!s} is referring "
+                f"the following missing modules {missing_packages}."
+            )
+
+    # Selected modules does not exists
+    if missing_modules := set(selected_modules) - modules.available:
+        hint = ModuleDefinition.long(missing_modules, organization_dir)
+        raise ToolkitMissingModuleError(
+            f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
+        )
+
+    # Nothing is Selected
+    if not modules.selected:
+        raise ToolkitEnvError(
+            f"No selected modules specified in {config.filepath!s}, have you configured "
+            f"the environment ({config.environment.name})?"
+        )
+
+    # Dev modules warning (non-critical)
+    dev_modules = modules.available_names & DEV_ONLY_MODULES
+    if dev_modules and config.environment.validation_type != "dev":
+        warnings.append(
+            MediumSeverityWarning(
+                "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
+                f"{humanize_collection(dev_modules)!r}",
+            )
+        )
+
+    return warnings

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.97
+      image: cognite/toolkit:0.7.30
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.97
+      image: cognite/toolkit:0.7.30
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,13 +4,14 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.97"
+version = "0.7.30"
 
-[alpha_flags]
-external-libraries = true
 
 [plugins]
-run = true
+dump = false
+dev = false
+data = false
+
 
 [library.toolkit-data]
 url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.97"
+__version__ = "0.7.30"

cognite_toolkit/config.dev.yaml

@@ -0,0 +1,13 @@
+environment:
+  name: dev
+  project: infield-experimental-test-aws
+  validation-type: dev
+  selected:
+    - modules/eriks-location
+
+variables:
+  modules:
+    eriks-location:
+      - location: erik-infield-cdm2
+        location_name: Oslo CDM - Erik tookit
+        location_description: Used to test toolkit ingestion for InField cdm

{cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.30.dist-info}/METADATA

@@ -1,38 +1,38 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.97
+Version: 0.7.30
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
-Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
-Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
-Project-URL: GitHub, https://github.com/cognitedata/toolkit
-Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>
 License-Expression: Apache-2.0
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: cognite-sdk<8.0.0,>=7.83.0
-Requires-Dist: filelock>=3.18.0
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: mixpanel>=4.10.1
-Requires-Dist: packaging>=25
-Requires-Dist: pandas<3.0.0,>=1.5.3
-Requires-Dist: pip>=25.0.1
-Requires-Dist: pydantic>=2.11.0
-Requires-Dist: python-dateutil>=2.9.0
 Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: cognite-sdk>=7.87.0,<8.0.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pandas>=1.5.3,<3.0.0
 Requires-Dist: pyyaml>=6.0.1
-Requires-Dist: questionary>=2.0.1
+Requires-Dist: typer>=0.12.0,<1.0.0
 Requires-Dist: rich>=13.9.4
-Requires-Dist: sentry-sdk>=2.1.0
-Requires-Dist: toml>=0.10.2
-Requires-Dist: tomli<3.0.0,>=2.0.1; python_version < '3.11'
-Requires-Dist: typer<1.0.0,>=0.12.0
+Requires-Dist: questionary>=2.0.1
+Requires-Dist: tomli>=2.0.1,<3.0.0 ; python_full_version < '3.11'
+Requires-Dist: packaging>=25
 Requires-Dist: typing-extensions>=4.0.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: sentry-sdk>=2.1.0
+Requires-Dist: mixpanel>=4.10.1
+Requires-Dist: pydantic>=2.11.0
+Requires-Dist: python-dateutil>=2.9.0
+Requires-Dist: pip>=25.0.1
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: sqlparse>=0.5.3 ; extra == 'sql'
+Requires-Dist: pyarrow>=20.0.0 ; extra == 'table'
+Requires-Dist: openpyxl>=3.1.5 ; extra == 'table'
+Requires-Python: >=3.10
+Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
+Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Project-URL: GitHub, https://github.com/cognitedata/toolkit
+Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Provides-Extra: sql
-Requires-Dist: sqlparse>=0.5.3; extra == 'sql'
 Provides-Extra: table
-Requires-Dist: openpyxl>=3.1.5; extra == 'table'
-Requires-Dist: pyarrow>=20.0.0; extra == 'table'
 Description-Content-Type: text/markdown
 
 # Cognite Data Fusion Toolkit