cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py

@@ -15,7 +15,6 @@
 
 import json
 from collections.abc import Hashable, Iterable, Sequence
-from functools import lru_cache
 from graphlib import CycleError, TopologicalSorter
 from pathlib import Path
 from typing import Any, final
@@ -23,30 +22,28 @@ from typing import Any, final
 from cognite.client.data_classes import (
     ClientCredentials,
     Workflow,
-    WorkflowList,
     WorkflowTrigger,
-    WorkflowTriggerList,
     WorkflowTriggerUpsert,
-    WorkflowTriggerUpsertList,
     WorkflowUpsert,
-    WorkflowUpsertList,
     WorkflowVersion,
     WorkflowVersionId,
-    WorkflowVersionList,
     WorkflowVersionUpsert,
-    WorkflowVersionUpsertList,
 )
 from cognite.client.data_classes.capabilities import (
     Capability,
     WorkflowOrchestrationAcl,
 )
-from cognite.client.data_classes.workflows import SubworkflowReferenceParameters
+from cognite.client.data_classes.workflows import (
+    SubworkflowReferenceParameters,
+    WorkflowList,
+    WorkflowTriggerList,
+    WorkflowVersionList,
+)
 from cognite.client.exceptions import CogniteAuthError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich import print
 from rich.console import Console
 
-from cognite_toolkit._cdf_tk._parameters import ANY_INT, ANY_STR, ANYTHING, ParameterSpec, ParameterSpecSet
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
@@ -64,7 +61,7 @@ from cognite_toolkit._cdf_tk.utils import (
     calculate_secure_hash,
     humanize_collection,
     load_yaml_inject_variables,
-    to_directory_compatible,
+    sanitize_filename,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import read_auth, try_find_error
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable
@@ -77,13 +74,10 @@ from .transformation import TransformationCRUD
 
 
 @final
-class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertList, WorkflowList]):
+class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow]):
     folder_name = "workflows"
-    filename_pattern = r"^.*Workflow$"
     resource_cls = Workflow
     resource_write_cls = WorkflowUpsert
-    list_cls = WorkflowList
-    list_write_cls = WorkflowUpsertList
     kind = "Workflow"
     dependencies = frozenset(
         {
@@ -142,18 +136,14 @@ class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertLis
             dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
         return dumped
 
-    def retrieve(self, ids: SequenceNotStr[str]) -> WorkflowList:
+    def retrieve(self, ids: SequenceNotStr[str]) -> Sequence[Workflow]:
         return self.client.workflows.retrieve(external_id=ids, ignore_unknown_ids=True)
 
-    def _upsert(self, items: WorkflowUpsert | WorkflowUpsertList) -> WorkflowList:
-        upserts = [items] if isinstance(items, WorkflowUpsert) else items
-        return self.client.workflows.upsert(upserts)
+    def create(self, items: Sequence[WorkflowUpsert]) -> WorkflowList:
+        return self.client.workflows.upsert(items)
 
-    def create(self, items: WorkflowUpsert | WorkflowUpsertList) -> WorkflowList:
-        return self._upsert(items)
-
-    def update(self, items: WorkflowUpsertList) -> WorkflowList:
-        return self._upsert(items)
+    def update(self, items: Sequence[WorkflowUpsert]) -> WorkflowList:
+        return self.client.workflows.upsert(items)
 
     def delete(self, ids: SequenceNotStr[str]) -> int:
         successes = 0
@@ -182,28 +172,6 @@ class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertLis
             if workflow.data_set_id == data_set.id:
                 yield workflow
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        spec.add(
-            ParameterSpec(
-                ("dataSetExternalId",),
-                frozenset({"str"}),
-                is_required=False,
-                _is_nullable=True,
-            )
-        )
-        spec.discard(
-            ParameterSpec(
-                ("dataSetId",),
-                frozenset({"str"}),
-                is_required=False,
-                _is_nullable=True,
-            )
-        )
-        return spec
-
     @classmethod
     def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         """Returns all items that this item requires.
@@ -216,17 +184,10 @@ class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertLis
 
 
 @final
-class WorkflowVersionCRUD(
-    ResourceCRUD[
-        WorkflowVersionId, WorkflowVersionUpsert, WorkflowVersion, WorkflowVersionUpsertList, WorkflowVersionList
-    ]
-):
+class WorkflowVersionCRUD(ResourceCRUD[WorkflowVersionId, WorkflowVersionUpsert, WorkflowVersion]):
     folder_name = "workflows"
-    filename_pattern = r"^.*WorkflowVersion$"
     resource_cls = WorkflowVersion
     resource_write_cls = WorkflowVersionUpsert
-    list_cls = WorkflowVersionList
-    list_write_cls = WorkflowVersionUpsertList
     kind = "WorkflowVersion"
     dependencies = frozenset({WorkflowCRUD})
     parent_resource = frozenset({WorkflowCRUD})
@@ -422,16 +383,16 @@ class WorkflowVersionCRUD(
             return WorkflowVersionList([])
         return self.client.workflows.versions.retrieve(workflow_external_id=list(ids), ignore_unknown_ids=True)
 
-    def _upsert(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList:
+    def _upsert(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
         return self.client.workflows.versions.upsert(items)
 
-    def create(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList:
+    def create(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
         upserted: list[WorkflowVersion] = []
         for item in self.topological_sort(items):
             upserted.extend(self.client.workflows.versions.upsert([item]))
         return WorkflowVersionList(upserted)
 
-    def update(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList:
+    def update(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
         return self._upsert(items)
 
     def delete(self, ids: SequenceNotStr[WorkflowVersionId]) -> int:
@@ -454,48 +415,6 @@ class WorkflowVersionCRUD(
         workflow_ids = [parent_id for parent_id in parent_ids if isinstance(parent_id, str)] if parent_ids else None
         return self.client.workflows.versions.list(limit=-1, workflow_version_ids=workflow_ids)  # type: ignore[arg-type]
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # The Parameter class in the SDK class WorkflowVersion implementation is deviating from the API.
-        # So we need to modify the spec to match the API.
-        parameter_path = ("workflowDefinition", "tasks", ANY_INT, "parameters")
-        length = len(parameter_path)
-        for item in spec:
-            if len(item.path) >= length + 1 and item.path[:length] == parameter_path[:length]:
-                # Add extra ANY_STR layer
-                # The spec class is immutable, so we use this trick to modify it.
-                object.__setattr__(item, "path", (*item.path[:length], ANY_STR, *item.path[length:]))
-        spec.add(ParameterSpec((*parameter_path, ANY_STR), frozenset({"dict"}), is_required=True, _is_nullable=False))
-        # The depends on is implemented as a list of string in the SDK, but in the API spec it
-        # is a list of objects with one 'externalId' field.
-        spec.add(
-            ParameterSpec(
-                ("workflowDefinition", "tasks", ANY_INT, "dependsOn", ANY_INT, "externalId"),
-                frozenset({"str"}),
-                is_required=False,
-                _is_nullable=False,
-            )
-        )
-        spec.add(
-            ParameterSpec(
-                ("workflowDefinition", "tasks", ANY_INT, "type"),
-                frozenset({"str"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        spec.add(
-            ParameterSpec(
-                ("workflowDefinition", "tasks", ANY_INT, "parameters", "subworkflow", ANYTHING),
-                frozenset({"dict"}),
-                is_required=False,
-                _is_nullable=False,
-            )
-        )
-        return spec
-
     @classmethod
     def as_str(cls, id: WorkflowVersionId) -> str:
         if id.version is None:
@@ -505,7 +424,7 @@ class WorkflowVersionCRUD(
         else:
             version = f"_v{id.version}"
 
-        return to_directory_compatible(f"{id.workflow_external_id}{version}")
+        return sanitize_filename(f"{id.workflow_external_id}{version}")
 
     @classmethod
     def topological_sort(cls, items: Sequence[WorkflowVersionUpsert]) -> list[WorkflowVersionUpsert]:
@@ -533,15 +452,10 @@ class WorkflowVersionCRUD(
 
 
 @final
-class WorkflowTriggerCRUD(
-    ResourceCRUD[str, WorkflowTriggerUpsert, WorkflowTrigger, WorkflowTriggerUpsertList, WorkflowTriggerList]
-):
+class WorkflowTriggerCRUD(ResourceCRUD[str, WorkflowTriggerUpsert, WorkflowTrigger]):
     folder_name = "workflows"
-    filename_pattern = r"^.*WorkflowTrigger$"
     resource_cls = WorkflowTrigger
     resource_write_cls = WorkflowTriggerUpsert
-    list_cls = WorkflowTriggerList
-    list_write_cls = WorkflowTriggerUpsertList
     kind = "WorkflowTrigger"
     dependencies = frozenset({WorkflowCRUD, WorkflowVersionCRUD, GroupResourceScopedCRUD, GroupAllScopedCRUD})
     parent_resource = frozenset({WorkflowCRUD})
@@ -588,10 +502,10 @@ class WorkflowTriggerCRUD(
             WorkflowOrchestrationAcl.Scope.All(),
         )
 
-    def create(self, items: WorkflowTriggerUpsertList) -> WorkflowTriggerList:
+    def create(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
         return self._upsert(items)
 
-    def _upsert(self, items: WorkflowTriggerUpsertList) -> WorkflowTriggerList:
+    def _upsert(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
         created = WorkflowTriggerList([])
         for item in items:
             created.append(self._upsert_item(item))
@@ -611,7 +525,7 @@ class WorkflowTriggerCRUD(
         lookup = set(ids)
         return WorkflowTriggerList([trigger for trigger in all_triggers if trigger.external_id in lookup])
 
-    def update(self, items: WorkflowTriggerUpsertList) -> WorkflowTriggerList:
+    def update(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
         return self._upsert(items)
 
     def delete(self, ids: SequenceNotStr[str]) -> int:
@@ -638,45 +552,6 @@ class WorkflowTriggerCRUD(
             return (trigger for trigger in triggers if trigger.workflow_external_id in workflow_ids)
         return triggers
 
-    @classmethod
-    @lru_cache(maxsize=1)
-    def get_write_cls_parameter_spec(cls) -> ParameterSpecSet:
-        spec = super().get_write_cls_parameter_spec()
-        # Removed by the SDK
-        spec.add(
-            ParameterSpec(
-                ("triggerRule", "triggerType"),
-                frozenset({"str"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        spec.add(
-            ParameterSpec(
-                ("authentication",),
-                frozenset({"dict"}),
-                is_required=True,
-                _is_nullable=False,
-            )
-        )
-        spec.add(
-            ParameterSpec(
-                ("authentication", "clientId"),
-                frozenset({"str"}),
-                is_required=False,
-                _is_nullable=False,
-            )
-        )
-        spec.add(
-            ParameterSpec(
-                ("authentication", "clientSecret"),
-                frozenset({"str"}),
-                is_required=False,
-                _is_nullable=False,
-            )
-        )
-        return spec
-
     @classmethod
     def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
         """Returns all items that this item requires."""

cognite_toolkit/_cdf_tk/cruds/_worker.py

@@ -1,17 +1,12 @@
 import re
 import warnings
-from collections.abc import Hashable
+from collections.abc import Hashable, Sequence
 from copy import deepcopy
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, cast
 
 from cognite.client.data_classes import FunctionWrite
-from cognite.client.data_classes._base import (
-    T_CogniteResourceList,
-    T_WritableCogniteResource,
-    T_WriteClass,
-)
 from cognite.client.data_classes.capabilities import Capability
 from rich import print
 from rich.panel import Panel
@@ -19,32 +14,29 @@ from yaml import YAMLError
 
 from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS
 from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError, ToolkitYAMLFormatError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings
 from cognite_toolkit._cdf_tk.utils import to_diff
 
 from . import FunctionCRUD
-from ._base_cruds import T_ID, ResourceCRUD, T_WritableCogniteResourceList
+from ._base_cruds import T_ID, ResourceCRUD, T_ResourceRequest, T_ResourceResponse
 
 if TYPE_CHECKING:
     from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
 
 
 @dataclass
-class CategorizedResources(Generic[T_ID, T_CogniteResourceList]):
-    to_create: T_CogniteResourceList
-    to_update: T_CogniteResourceList
-    to_delete: list[T_ID]
-    unchanged: T_CogniteResourceList
+class CategorizedResources(Generic[T_ID, T_ResourceRequest]):
+    to_create: list[T_ResourceRequest] = field(default_factory=list)
+    to_update: list[T_ResourceRequest] = field(default_factory=list)
+    to_delete: list[T_ID] = field(default_factory=list)
+    unchanged: list[T_ResourceRequest] = field(default_factory=list)
 
 
-class ResourceWorker(
-    Generic[T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList]
-):
+class ResourceWorker(Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
     def __init__(
         self,
-        loader: ResourceCRUD[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         action: str,
     ):
         self.loader = loader
@@ -56,12 +48,16 @@ class ResourceWorker(
     ) -> list[Path]:
         filepaths = self.loader.find_files(directory)
 
-        for read_module in read_modules or []:
-            if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
-                # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
-                # These table formats are not built, i.e., no variable replacement is done,
-                # so we load them directly from the source module.
-                filepaths.extend(self.loader.find_files(resource_dir, include_formats=TABLE_FORMATS))
+        if not Flags.v08.is_enabled():
+            for read_module in read_modules or []:
+                if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
+                    # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
+                    # These table formats are not built, i.e., no variable replacement is done,
+                    # so we load them directly from the source module.
+                    table_files = [
+                        file for file in self.loader.find_files(resource_dir) if file.suffix in TABLE_FORMATS
+                    ]
+                    filepaths.extend(table_files)
 
         if not sort:
             return filepaths
@@ -102,14 +98,13 @@ class ResourceWorker(
         self.validate_access(local_by_id, is_dry_run)
 
         # Lookup the existing resources in CDF
-        cdf_resources: T_WritableCogniteResourceList
         cdf_resources = self.loader.retrieve(list(local_by_id.keys()))
         return self.categorize_resources(local_by_id, cdf_resources, force_update, verbose)
 
     def load_resources(
         self, filepaths: list[Path], environment_variables: dict[str, str | None] | None, is_dry_run: bool
-    ) -> dict[T_ID, tuple[dict[str, Any], T_WriteClass]]:
-        local_by_id: dict[T_ID, tuple[dict[str, Any], T_WriteClass]] = {}
+    ) -> dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]]:
+        local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]] = {}
         # Load all resources from files, get ids, and remove duplicates.
         environment_variables = environment_variables or {}
         for filepath in filepaths:
@@ -145,7 +140,9 @@ class ResourceWorker(
                     warning.print_warning()
         return local_by_id
 
-    def validate_access(self, local_by_id: dict[T_ID, tuple[dict[str, Any], T_WriteClass]], is_dry_run: bool) -> None:
+    def validate_access(
+        self, local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]], is_dry_run: bool
+    ) -> None:
         capabilities: Capability | list[Capability]
         if isinstance(self.loader, FunctionCRUD):
             function_loader: FunctionCRUD = self.loader
@@ -160,17 +157,12 @@ class ResourceWorker(
 
     def categorize_resources(
         self,
-        local_by_id: dict[T_ID, tuple[dict[str, Any], T_WriteClass]],
-        cdf_resources: T_WritableCogniteResourceList,
+        local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]],
+        cdf_resources: Sequence[T_ResourceResponse],
         force_update: bool,
         verbose: bool,
-    ) -> CategorizedResources:
-        resources: CategorizedResources[T_ID, T_CogniteResourceList] = CategorizedResources(
-            to_create=self.loader.list_write_cls([]),
-            to_update=self.loader.list_write_cls([]),
-            to_delete=[],
-            unchanged=self.loader.list_write_cls([]),
-        )
+    ) -> CategorizedResources[T_ID, T_ResourceRequest]:
+        resources: CategorizedResources[T_ID, T_ResourceRequest] = CategorizedResources()
         cdf_resource_by_id = {self.loader.get_id(resource): resource for resource in cdf_resources}
         for identifier, (local_dict, local_resource) in local_by_id.items():
             cdf_resource = cdf_resource_by_id.get(identifier)

cognite_toolkit/_cdf_tk/data_classes/__init__.py

@@ -32,6 +32,7 @@ from ._deploy_results import (
 from ._module_directories import ModuleDirectories, ModuleLocation
 from ._module_resources import ModuleResources
 from ._packages import Package, Packages
+from ._tracking_info import CommandTrackingInfo
 from ._yaml_comments import YAMLComments
 
 __all__ = [
@@ -47,6 +48,7 @@ __all__ = [
     "BuiltResource",
     "BuiltResourceFull",
     "BuiltResourceList",
+    "CommandTrackingInfo",
     "ConfigEntry",
     "ConfigYAMLs",
     "DatapointDeployResult",

cognite_toolkit/_cdf_tk/data_classes/_build_variables.py

@@ -8,11 +8,11 @@ from functools import cached_property
 from pathlib import Path
 from typing import Any, Literal, SupportsIndex, overload
 
+from cognite_toolkit._cdf_tk.cruds._resource_cruds.transformation import TransformationCRUD
+from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.feature_flags import Flags
 
-from ._module_directories import ModuleLocation
-
 if sys.version_info >= (3, 11):
     from typing import Self
 else:
@@ -161,16 +161,19 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
         ]
 
     @overload
-    def replace(self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[False] = False) -> str: ...
+    def replace(self, content: str, file_path: Path | None = None, use_placeholder: Literal[False] = False) -> str: ...
 
     @overload
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[True] = True
+        self, content: str, file_path: Path | None = None, use_placeholder: Literal[True] = True
     ) -> tuple[str, dict[str, BuildVariable]]: ...
 
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: bool = False
+        self, content: str, file_path: Path | None = None, use_placeholder: bool = False
     ) -> str | tuple[str, dict[str, BuildVariable]]:
+        # Extract file suffix from path, default to .yaml if not provided
+        file_suffix = file_path.suffix if file_path and file_path.suffix else ".yaml"
+
         variable_by_placeholder: dict[str, BuildVariable] = {}
         for variable in self:
             if not use_placeholder:
@@ -180,22 +183,125 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
                 variable_by_placeholder[replace] = variable
 
             _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}"
-            if file_suffix in {".yaml", ".yml", ".json"}:
-                # Preserve data types
-                pattern = _core_pattern
-                if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
-                    replace = f'"{replace}"'
-                    pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
-                elif replace is None:
-                    replace = "null"
-                content = re.sub(pattern, str(replace), content)
+            if file_suffix == ".sql":
+                # For SQL files, convert lists to SQL-style tuples
+                if isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                content = re.sub(_core_pattern, str(replace), content)
+            elif file_suffix in {".yaml", ".yml", ".json"}:
+                # Check if this is a transformation file (ends with Transformation.yaml/yml)
+                is_transformation_file = file_path is not None and f".{TransformationCRUD.kind}." in file_path.name
+                # Check if variable is within a query field (SQL context)
+                is_in_query_field = self._is_in_query_field(content, variable.key)
+
+                # For lists in query fields, use SQL-style tuples
+                # For transformation files, ensure SQL conversion is applied to query property variables
+                if is_transformation_file and is_in_query_field and isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                    # Use simple pattern for SQL context (no YAML quoting needed)
+                    content = re.sub(_core_pattern, str(replace), content)
+                else:
+                    # Preserve data types for YAML
+                    pattern = _core_pattern
+                    if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
+                        replace = f'"{replace}"'
+                        pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
+                    elif replace is None:
+                        replace = "null"
+                    content = re.sub(pattern, str(replace), content)
             else:
+                # For other file types, use simple string replacement
                 content = re.sub(_core_pattern, str(replace), content)
         if use_placeholder:
             return content, variable_by_placeholder
         else:
             return content
 
+    @staticmethod
+    def _is_transformation_file(file_path: Path) -> bool:
+        """Check if the file path indicates a transformation YAML file.
+
+        Transformation files are YAML files in the "transformations" folder.
+
+        Args:
+            file_path: The file path to check
+
+        Returns:
+            True if the file is a transformation YAML file
+        """
+        # Check if path contains "transformations" folder and ends with .yaml/.yml
+        path_str = file_path.as_posix().lower()
+        return "transformations" in path_str and file_path.suffix.lower() in {".yaml", ".yml"}
+
+    @staticmethod
+    def _format_list_as_sql_tuple(replace: list[Any]) -> str:
+        """Format a list as a SQL-style tuple string.
+
+        Args:
+            replace: The list to format
+
+        Returns:
+            SQL tuple string, e.g., "('A', 'B', 'C')" or "()" for empty lists
+        """
+        if not replace:
+            # Empty list becomes empty SQL tuple
+            return "()"
+        else:
+            # Format list as SQL tuple: ('A', 'B', 'C')
+            formatted_items = []
+            for item in replace:
+                if item is None:
+                    formatted_items.append("NULL")
+                elif isinstance(item, str):
+                    formatted_items.append(f"'{item}'")
+                else:
+                    formatted_items.append(str(item))
+            return f"({', '.join(formatted_items)})"
+
+    @staticmethod
+    def _is_in_query_field(content: str, variable_key: str) -> bool:
+        """Check if a variable is within a query field in YAML.
+
+        Assumes query is a top-level property. This detects various YAML formats:
+        - query: >-
+        - query: |
+        - query: "..."
+        - query: ...
+        """
+        lines = content.split("\n")
+        variable_pattern = rf"{{{{\s*{re.escape(variable_key)}\s*}}}}"
+        in_query_field = False
+
+        for line in lines:
+            # Check if this line starts a top-level query field
+            query_match = re.match(r"^query\s*:\s*(.*)$", line)
+            if query_match:
+                in_query_field = True
+                query_content_start = query_match.group(1).strip()
+
+                # Check if variable is on the same line as query: declaration
+                if re.search(variable_pattern, line):
+                    return True
+
+                # If query content starts on same line (not a block scalar), check it
+                if query_content_start and not query_content_start.startswith(("|", ">", "|-", ">-", "|+", ">+")):
+                    if re.search(variable_pattern, query_content_start):
+                        return True
+                continue
+
+            # Check if we're still in the query field
+            if in_query_field:
+                # If we hit another top-level property, we've exited the query field
+                if re.match(r"^\w+\s*:", line):
+                    in_query_field = False
+                    continue
+
+                # We're still in the query field, check for variable
+                if re.search(variable_pattern, line):
+                    return True
+
+        return False
+
     # Implemented to get correct type hints
     def __iter__(self) -> Iterator[BuildVariable]:
         return super().__iter__()

cognite_toolkit/_cdf_tk/data_classes/_built_modules.py

@@ -30,6 +30,8 @@ class BuiltModule:
     warning_count: int
     status: str
     iteration: int
+    package_id: str | None = None
+    module_id: str | None = None
 
     @classmethod
     def load(cls, data: dict[str, Any]) -> Self:
@@ -44,6 +46,8 @@ class BuiltModule:
             warning_count=data.get("warning_count", 0),
             status=data.get("status", "Success"),
             iteration=data.get("iteration", 1),
+            package_id=data.get("package_id"),
+            module_id=data.get("module_id"),
         )
 
     def dump(self) -> dict[str, Any]:
@@ -57,6 +61,8 @@ class BuiltModule:
             "warning_count": self.warning_count,
             "status": self.status,
             "iteration": self.iteration,
+            "package_id": self.package_id,
+            "module_id": self.module_id,
         }
 
 

cognite_toolkit/_cdf_tk/data_classes/_built_resources.py

@@ -158,7 +158,7 @@ class BuiltResourceFull(BuiltResource[T_ID]):
     def load_resource_dict(
         self, environment_variables: dict[str, str | None], validate: bool = False
     ) -> dict[str, Any]:
-        content = self.build_variables.replace(safe_read(self.source.path))
+        content = self.build_variables.replace(safe_read(self.source.path), self.source.path)
         loader = cast(ResourceCRUD, get_crud(self.resource_dir, self.kind))
         raw = load_yaml_inject_variables(
             content,