cognite-toolkit 0.6.89__py3-none-any.whl → 0.6.91__py3-none-any.whl

cognite_toolkit/_cdf_tk/data_classes/_build_variables.py

@@ -8,11 +8,11 @@ from functools import cached_property
 from pathlib import Path
 from typing import Any, Literal, SupportsIndex, overload
 
+from cognite_toolkit._cdf_tk.cruds._resource_cruds.transformation import TransformationCRUD
+from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.feature_flags import Flags
 
-from ._module_directories import ModuleLocation
-
 if sys.version_info >= (3, 11):
     from typing import Self
 else:
@@ -161,16 +161,19 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
         ]
 
     @overload
-    def replace(self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[False] = False) -> str: ...
+    def replace(self, content: str, file_path: Path | None = None, use_placeholder: Literal[False] = False) -> str: ...
 
     @overload
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[True] = True
+        self, content: str, file_path: Path | None = None, use_placeholder: Literal[True] = True
     ) -> tuple[str, dict[str, BuildVariable]]: ...
 
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: bool = False
+        self, content: str, file_path: Path | None = None, use_placeholder: bool = False
     ) -> str | tuple[str, dict[str, BuildVariable]]:
+        # Extract file suffix from path, default to .yaml if not provided
+        file_suffix = file_path.suffix if file_path and file_path.suffix else ".yaml"
+
         variable_by_placeholder: dict[str, BuildVariable] = {}
         for variable in self:
             if not use_placeholder:
@@ -180,22 +183,125 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
                 variable_by_placeholder[replace] = variable
 
             _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}"
-            if file_suffix in {".yaml", ".yml", ".json"}:
-                # Preserve data types
-                pattern = _core_pattern
-                if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
-                    replace = f'"{replace}"'
-                    pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
-                elif replace is None:
-                    replace = "null"
-                content = re.sub(pattern, str(replace), content)
+            if file_suffix == ".sql":
+                # For SQL files, convert lists to SQL-style tuples
+                if isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                content = re.sub(_core_pattern, str(replace), content)
+            elif file_suffix in {".yaml", ".yml", ".json"}:
+                # Check if this is a transformation file (ends with Transformation.yaml/yml)
+                is_transformation_file = file_path is not None and f".{TransformationCRUD.kind}." in file_path.name
+                # Check if variable is within a query field (SQL context)
+                is_in_query_field = self._is_in_query_field(content, variable.key)
+
+                # For lists in query fields, use SQL-style tuples
+                # For transformation files, ensure SQL conversion is applied to query property variables
+                if is_transformation_file and is_in_query_field and isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                    # Use simple pattern for SQL context (no YAML quoting needed)
+                    content = re.sub(_core_pattern, str(replace), content)
+                else:
+                    # Preserve data types for YAML
+                    pattern = _core_pattern
+                    if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
+                        replace = f'"{replace}"'
+                        pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
+                    elif replace is None:
+                        replace = "null"
+                    content = re.sub(pattern, str(replace), content)
             else:
+                # For other file types, use simple string replacement
                 content = re.sub(_core_pattern, str(replace), content)
         if use_placeholder:
             return content, variable_by_placeholder
         else:
             return content
 
+    @staticmethod
+    def _is_transformation_file(file_path: Path) -> bool:
+        """Check if the file path indicates a transformation YAML file.
+
+        Transformation files are YAML files in the "transformations" folder.
+
+        Args:
+            file_path: The file path to check
+
+        Returns:
+            True if the file is a transformation YAML file
+        """
+        # Check if path contains "transformations" folder and ends with .yaml/.yml
+        path_str = file_path.as_posix().lower()
+        return "transformations" in path_str and file_path.suffix.lower() in {".yaml", ".yml"}
+
+    @staticmethod
+    def _format_list_as_sql_tuple(replace: list[Any]) -> str:
+        """Format a list as a SQL-style tuple string.
+
+        Args:
+            replace: The list to format
+
+        Returns:
+            SQL tuple string, e.g., "('A', 'B', 'C')" or "()" for empty lists
+        """
+        if not replace:
+            # Empty list becomes empty SQL tuple
+            return "()"
+        else:
+            # Format list as SQL tuple: ('A', 'B', 'C')
+            formatted_items = []
+            for item in replace:
+                if item is None:
+                    formatted_items.append("NULL")
+                elif isinstance(item, str):
+                    formatted_items.append(f"'{item}'")
+                else:
+                    formatted_items.append(str(item))
+            return f"({', '.join(formatted_items)})"
+
+    @staticmethod
+    def _is_in_query_field(content: str, variable_key: str) -> bool:
+        """Check if a variable is within a query field in YAML.
+
+        Assumes query is a top-level property. This detects various YAML formats:
+        - query: >-
+        - query: |
+        - query: "..."
+        - query: ...
+        """
+        lines = content.split("\n")
+        variable_pattern = rf"{{{{\s*{re.escape(variable_key)}\s*}}}}"
+        in_query_field = False
+
+        for line in lines:
+            # Check if this line starts a top-level query field
+            query_match = re.match(r"^query\s*:\s*(.*)$", line)
+            if query_match:
+                in_query_field = True
+                query_content_start = query_match.group(1).strip()
+
+                # Check if variable is on the same line as query: declaration
+                if re.search(variable_pattern, line):
+                    return True
+
+                # If query content starts on same line (not a block scalar), check it
+                if query_content_start and not query_content_start.startswith(("|", ">", "|-", ">-", "|+", ">+")):
+                    if re.search(variable_pattern, query_content_start):
+                        return True
+                continue
+
+            # Check if we're still in the query field
+            if in_query_field:
+                # If we hit another top-level property, we've exited the query field
+                if re.match(r"^\w+\s*:", line):
+                    in_query_field = False
+                    continue
+
+                # We're still in the query field, check for variable
+                if re.search(variable_pattern, line):
+                    return True
+
+        return False
+
     # Implemented to get correct type hints
     def __iter__(self) -> Iterator[BuildVariable]:
         return super().__iter__()

cognite_toolkit/_cdf_tk/data_classes/_built_resources.py

@@ -158,7 +158,7 @@ class BuiltResourceFull(BuiltResource[T_ID]):
     def load_resource_dict(
         self, environment_variables: dict[str, str | None], validate: bool = False
     ) -> dict[str, Any]:
-        content = self.build_variables.replace(safe_read(self.source.path))
+        content = self.build_variables.replace(safe_read(self.source.path), self.source.path)
         loader = cast(ResourceCRUD, get_crud(self.resource_dir, self.kind))
         raw = load_yaml_inject_variables(
             content,

cognite_toolkit/_cdf_tk/protocols.py

@@ -0,0 +1,97 @@
+import sys
+from collections.abc import Collection, Iterator
+from typing import Any, Generic, Protocol, TypeVar
+
+from cognite.client import CogniteClient
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class ResourceRequestProtocol(Protocol):
+    @classmethod
+    def _load(cls, data: dict[str, Any]) -> Self: ...
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]: ...
+
+
+class ResourceResponseProtocol(Protocol):
+    def as_write(self) -> ResourceRequestProtocol: ...
+
+
+T_ResourceRequest = TypeVar("T_ResourceRequest", bound=ResourceRequestProtocol)
+T_ResourceResponse = TypeVar("T_ResourceResponse", bound=ResourceResponseProtocol)
+
+
+class ResourceRequestListProtocol(Protocol, Generic[T_ResourceRequest]):
+    def __init__(self, collection: Collection[T_ResourceRequest]): ...
+
+    def __getitem__(self, index: int) -> T_ResourceRequest: ...
+
+    def __setitem__(self, index: int, value: T_ResourceRequest) -> None: ...
+
+    def __delitem__(self, index: int) -> None: ...
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[T_ResourceRequest]: ...
+
+    def __contains__(self, value: object) -> bool: ...
+
+    def __reversed__(self) -> Iterator[T_ResourceRequest]: ...
+
+    def insert(self, index: int, value: T_ResourceRequest) -> None: ...
+
+    def append(self, value: T_ResourceRequest) -> None: ...
+
+    def extend(self, values: Collection[T_ResourceRequest]) -> None: ...
+
+    def pop(self, index: int = -1) -> T_ResourceRequest: ...
+
+    def remove(self, value: T_ResourceRequest) -> None: ...
+
+    def clear(self) -> None: ...
+
+    @classmethod
+    def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> Self: ...
+
+
+class ResourceResponseListProtocol(Protocol, Generic[T_ResourceResponse]):
+    def __init__(self, collection: Collection[T_ResourceResponse]): ...
+
+    def __getitem__(self, index: int) -> T_ResourceResponse: ...
+
+    def __setitem__(self, index: int, value: T_ResourceResponse) -> None: ...
+
+    def __delitem__(self, index: int) -> None: ...
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[T_ResourceResponse]: ...
+
+    def __contains__(self, value: object) -> bool: ...
+
+    def __reversed__(self) -> Iterator[T_ResourceResponse]: ...
+
+    def insert(self, index: int, value: T_ResourceResponse) -> None: ...
+
+    def append(self, value: T_ResourceResponse) -> None: ...
+
+    def extend(self, values: Collection[T_ResourceResponse]) -> None: ...
+
+    def pop(self, index: int = -1) -> T_ResourceResponse: ...
+
+    def remove(self, value: T_ResourceResponse) -> None: ...
+
+    def clear(self) -> None: ...
+
+    def as_write(self) -> ResourceRequestListProtocol: ...
+
+    @classmethod
+    def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> Self: ...
+
+
+T_ResourceRequestList = TypeVar("T_ResourceRequestList", bound=ResourceRequestListProtocol)
+T_ResourceResponseList = TypeVar("T_ResourceResponseList", bound=ResourceResponseListProtocol)

cognite_toolkit/_cdf_tk/resource_classes/__init__.py

@@ -27,6 +27,7 @@ from .hosted_extractor_destination import HostedExtractorDestinationYAML
 from .hosted_extractor_job import HostedExtractorJobYAML
 from .hosted_extractor_mapping import HostedExtractorMappingYAML
 from .hosted_extractor_source import HostedExtractorSourceYAML
+from .infield_cdmv1 import InfieldLocationConfigYAML
 from .infield_v1 import InfieldV1YAML
 from .instance import EdgeYAML, NodeYAML
 from .labels import LabelsYAML
@@ -74,6 +75,7 @@ __all__ = [
     "HostedExtractorJobYAML",
     "HostedExtractorMappingYAML",
     "HostedExtractorSourceYAML",
+    "InfieldLocationConfigYAML",
     "InfieldV1YAML",
     "LabelsYAML",
     "LocationYAML",

cognite_toolkit/_cdf_tk/resource_classes/agent.py

@@ -55,3 +55,4 @@ class AgentYAML(ToolkitResource):
         "azure/gpt-4o-mini", description="The name of the model to use. Defaults to your CDF project's default model."
     )
     tools: list[AgentTool] | None = Field(None, description="A list of tools available to the agent.", max_length=20)
+    runtime_version: str | None = Field(None, description="The runtime version")

cognite_toolkit/_cdf_tk/resource_classes/infield_cdmv1.py

@@ -0,0 +1,94 @@
+from typing import Any
+
+from .base import BaseModelResource, ToolkitResource
+
+
+class ObservationFeatureToggles(BaseModelResource):
+    """Feature toggles for observations."""
+
+    is_enabled: bool | None = None
+    is_write_back_enabled: bool | None = None
+    notifications_endpoint_external_id: str | None = None
+    attachments_endpoint_external_id: str | None = None
+
+
+class FeatureToggles(BaseModelResource):
+    """Feature toggles for InField location configuration."""
+
+    three_d: bool | None = None
+    trends: bool | None = None
+    documents: bool | None = None
+    workorders: bool | None = None
+    notifications: bool | None = None
+    media: bool | None = None
+    template_checklist_flow: bool | None = None
+    workorder_checklist_flow: bool | None = None
+    observations: ObservationFeatureToggles | None = None
+
+
+class AccessManagement(BaseModelResource):
+    """Access management configuration."""
+
+    template_admins: list[str] | None = None  # list of CDF group external IDs
+    checklist_admins: list[str] | None = None  # list of CDF group external IDs
+
+
+class ResourceFilters(BaseModelResource):
+    """Resource filters."""
+
+    spaces: list[str] | None = None
+
+
+class RootLocationDataFilters(BaseModelResource):
+    """Data filters for root location."""
+
+    general: ResourceFilters | None = None
+    assets: ResourceFilters | None = None
+    files: ResourceFilters | None = None
+    timeseries: ResourceFilters | None = None
+
+
+class DataExplorationConfig(BaseModelResource):
+    """Properties for DataExplorationConfig node.
+
+    Contains configuration for data exploration features:
+    - observations: Observations feature configuration
+    - activities: Activities configuration
+    - documents: Document configuration
+    - notifications: Notifications configuration
+    - assets: Asset page configuration
+    """
+
+    space: str | None = None
+    external_id: str | None = None
+
+    observations: dict[str, Any] | None = None  # ObservationsConfigFeature
+    activities: dict[str, Any] | None = None  # ActivitiesConfiguration
+    documents: dict[str, Any] | None = None  # DocumentConfiguration
+    notifications: dict[str, Any] | None = None  # NotificationsConfiguration
+    assets: dict[str, Any] | None = None  # AssetPageConfiguration
+
+
+class InfieldLocationConfigYAML(ToolkitResource):
+    """Properties for InFieldLocationConfig node.
+
+    Currently migrated fields:
+    - root_location_external_id: Reference to the LocationFilterDTO external ID
+    - feature_toggles: Feature toggles migrated from old configuration
+    - rootAsset: Direct relation to the root asset (space and externalId)
+    - app_instance_space: Application instance space from appDataInstanceSpace
+    - access_management: Template and checklist admin groups (from templateAdmins and checklistAdmins)
+    - disciplines: List of disciplines (from disciplines in FeatureConfiguration)
+    - data_filters: Data filters for general, assets, files, and timeseries (from dataFilters in old configuration)
+    - data_exploration_config: Direct relation to the DataExplorationConfig node (shared across all locations)
+    """
+
+    space: str
+    external_id: str
+
+    root_location_external_id: str | None = None
+    feature_toggles: FeatureToggles | None = None
+    app_instance_space: str | None = None
+    access_management: AccessManagement | None = None
+    data_filters: RootLocationDataFilters | None = None
+    data_exploration_config: DataExplorationConfig | None = None

cognite_toolkit/_cdf_tk/utils/text.py

@@ -1,8 +1,10 @@
+import hashlib
 import re
 from collections.abc import Hashable
 
 from rich.console import Console
 
+from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
 
 INVALID_TITLE_REGEX = re.compile(r"[\\*?:/\[\]]")
@@ -90,3 +92,24 @@ def to_sentence_case(text: str) -> str:
     text = re.sub(r"(?<=[a-z])([A-Z])", r" \1", text)
     # Convert to lowercase
     return text.casefold()
+
+
+def sanitize_instance_external_id(external_id: str) -> str:
+    """Sanitize an instance external ID to be compatible with CDF requirements.
+
+    Args:
+        external_id: The external ID to sanitize.
+
+    Returns:
+        The sanitized external ID.
+    """
+    # CDF instance external IDs must be between 1 and 256 characters,
+    if not external_id or external_id == "\x00":
+        raise ToolkitValueError("External ID cannot be empty.")
+    elif len(external_id) <= 256:
+        return external_id
+    hasher = hashlib.sha256()
+    hasher.update(external_id.encode("utf-8"))
+    hash_digest = hasher.hexdigest()[:8]
+    sanitized_external_id = f"{external_id[:247]}_{hash_digest}"
+    return sanitized_external_id

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.89
+      image: cognite/toolkit:0.6.91
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.89
+      image: cognite/toolkit:0.6.91
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.89"
+version = "0.6.91"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.89"
+__version__ = "0.6.91"

{cognite_toolkit-0.6.89.dist-info → cognite_toolkit-0.6.91.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.89
+Version: 0.6.91
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases