cognite-toolkit 0.6.89__tar.gz → 0.6.90__tar.gz

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.89
+Version: 0.6.90
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cdf.toml

@@ -31,7 +31,7 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.89"
+version = "0.6.90"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -492,7 +492,7 @@ class BuildCommand(ToolkitCommand):
             # which is what we do in the deploy step to verify that the source file has not changed.
             source = SourceLocationEager(source_path, calculate_hash(source_path, shorten=True))
 
-            content = variables.replace(content, source_path.suffix)
+            content = variables.replace(content, source_path)
 
             replace_warnings = self._check_variables_replaced(content, module_dir, source_path)
 

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_cdf_tk/commands/pull.py

@@ -561,7 +561,7 @@ class PullCommand(ToolkitCommand):
 
             if has_changes and not dry_run:
                 new_content, extra_files = self._to_write_content(
-                    safe_read(source_file), to_write, resources, environment_variables, loader
+                    safe_read(source_file), to_write, resources, environment_variables, loader, source_file
                 )
                 with source_file.open("w", encoding=ENCODING, newline=NEWLINE) as f:
                     f.write(new_content)
@@ -649,6 +649,7 @@ class PullCommand(ToolkitCommand):
         loader: ResourceCRUD[
             T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
         ],
+        source_file: Path,
     ) -> tuple[str, dict[Path, str]]:
         # 1. Replace all variables with placeholders
         # 2. Load source and keep the comments
@@ -682,7 +683,7 @@ class PullCommand(ToolkitCommand):
                     variables_with_environment_list.append(variable)
             variables = BuildVariables(variables_with_environment_list)
 
-        content, value_by_placeholder = variables.replace(source, use_placeholder=True)
+        content, value_by_placeholder = variables.replace(source, source_file, use_placeholder=True)
         comments = YAMLComments.load(source)
         # If there is a variable in the identifier, we need to replace it with the value
         # such that we can look it up in the to_write dict.
@@ -690,10 +691,10 @@ class PullCommand(ToolkitCommand):
             # The safe read in ExtractionPipelineConfigLoader stringifies the config dict,
             # but we need to load it as a dict so we can write it back to the file maintaining
             # the order or the keys.
-            loaded = read_yaml_content(variables.replace(source))
+            loaded = read_yaml_content(variables.replace(source, source_file))
             loaded_with_placeholder = read_yaml_content(content)
         else:
-            loaded = read_yaml_content(loader.safe_read(variables.replace(source)))
+            loaded = read_yaml_content(loader.safe_read(variables.replace(source, source_file)))
             loaded_with_placeholder = read_yaml_content(loader.safe_read(content))
 
         built_by_identifier = {r.identifier: r for r in resources}
@@ -756,7 +757,7 @@ class PullCommand(ToolkitCommand):
             builder = create_builder(built.resource_dir, None)
             for extra in built.extra_sources:
                 extra_content, extra_placeholders = built.build_variables.replace(
-                    safe_read(extra.path), extra.path.suffix, use_placeholder=True
+                    safe_read(extra.path), extra.path, use_placeholder=True
                 )
                 key, _ = builder.load_extra_field(extra_content)
                 if key in item_write:

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py

@@ -8,11 +8,11 @@ from functools import cached_property
 from pathlib import Path
 from typing import Any, Literal, SupportsIndex, overload
 
+from cognite_toolkit._cdf_tk.cruds._resource_cruds.transformation import TransformationCRUD
+from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.feature_flags import Flags
 
-from ._module_directories import ModuleLocation
-
 if sys.version_info >= (3, 11):
     from typing import Self
 else:
@@ -161,16 +161,19 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
         ]
 
     @overload
-    def replace(self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[False] = False) -> str: ...
+    def replace(self, content: str, file_path: Path | None = None, use_placeholder: Literal[False] = False) -> str: ...
 
     @overload
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[True] = True
+        self, content: str, file_path: Path | None = None, use_placeholder: Literal[True] = True
     ) -> tuple[str, dict[str, BuildVariable]]: ...
 
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: bool = False
+        self, content: str, file_path: Path | None = None, use_placeholder: bool = False
     ) -> str | tuple[str, dict[str, BuildVariable]]:
+        # Extract file suffix from path, default to .yaml if not provided
+        file_suffix = file_path.suffix if file_path and file_path.suffix else ".yaml"
+
         variable_by_placeholder: dict[str, BuildVariable] = {}
         for variable in self:
             if not use_placeholder:
@@ -180,22 +183,125 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
                 variable_by_placeholder[replace] = variable
 
             _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}"
-            if file_suffix in {".yaml", ".yml", ".json"}:
-                # Preserve data types
-                pattern = _core_pattern
-                if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
-                    replace = f'"{replace}"'
-                    pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
-                elif replace is None:
-                    replace = "null"
-                content = re.sub(pattern, str(replace), content)
+            if file_suffix == ".sql":
+                # For SQL files, convert lists to SQL-style tuples
+                if isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                content = re.sub(_core_pattern, str(replace), content)
+            elif file_suffix in {".yaml", ".yml", ".json"}:
+                # Check if this is a transformation file (ends with Transformation.yaml/yml)
+                is_transformation_file = file_path is not None and f".{TransformationCRUD.kind}." in file_path.name
+                # Check if variable is within a query field (SQL context)
+                is_in_query_field = self._is_in_query_field(content, variable.key)
+
+                # For lists in query fields, use SQL-style tuples
+                # For transformation files, ensure SQL conversion is applied to query property variables
+                if is_transformation_file and is_in_query_field and isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                    # Use simple pattern for SQL context (no YAML quoting needed)
+                    content = re.sub(_core_pattern, str(replace), content)
+                else:
+                    # Preserve data types for YAML
+                    pattern = _core_pattern
+                    if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
+                        replace = f'"{replace}"'
+                        pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
+                    elif replace is None:
+                        replace = "null"
+                    content = re.sub(pattern, str(replace), content)
             else:
+                # For other file types, use simple string replacement
                 content = re.sub(_core_pattern, str(replace), content)
         if use_placeholder:
             return content, variable_by_placeholder
         else:
             return content
 
+    @staticmethod
+    def _is_transformation_file(file_path: Path) -> bool:
+        """Check if the file path indicates a transformation YAML file.
+
+        Transformation files are YAML files in the "transformations" folder.
+
+        Args:
+            file_path: The file path to check
+
+        Returns:
+            True if the file is a transformation YAML file
+        """
+        # Check if path contains "transformations" folder and ends with .yaml/.yml
+        path_str = file_path.as_posix().lower()
+        return "transformations" in path_str and file_path.suffix.lower() in {".yaml", ".yml"}
+
+    @staticmethod
+    def _format_list_as_sql_tuple(replace: list[Any]) -> str:
+        """Format a list as a SQL-style tuple string.
+
+        Args:
+            replace: The list to format
+
+        Returns:
+            SQL tuple string, e.g., "('A', 'B', 'C')" or "()" for empty lists
+        """
+        if not replace:
+            # Empty list becomes empty SQL tuple
+            return "()"
+        else:
+            # Format list as SQL tuple: ('A', 'B', 'C')
+            formatted_items = []
+            for item in replace:
+                if item is None:
+                    formatted_items.append("NULL")
+                elif isinstance(item, str):
+                    formatted_items.append(f"'{item}'")
+                else:
+                    formatted_items.append(str(item))
+            return f"({', '.join(formatted_items)})"
+
+    @staticmethod
+    def _is_in_query_field(content: str, variable_key: str) -> bool:
+        """Check if a variable is within a query field in YAML.
+
+        Assumes query is a top-level property. This detects various YAML formats:
+        - query: >-
+        - query: |
+        - query: "..."
+        - query: ...
+        """
+        lines = content.split("\n")
+        variable_pattern = rf"{{{{\s*{re.escape(variable_key)}\s*}}}}"
+        in_query_field = False
+
+        for line in lines:
+            # Check if this line starts a top-level query field
+            query_match = re.match(r"^query\s*:\s*(.*)$", line)
+            if query_match:
+                in_query_field = True
+                query_content_start = query_match.group(1).strip()
+
+                # Check if variable is on the same line as query: declaration
+                if re.search(variable_pattern, line):
+                    return True
+
+                # If query content starts on same line (not a block scalar), check it
+                if query_content_start and not query_content_start.startswith(("|", ">", "|-", ">-", "|+", ">+")):
+                    if re.search(variable_pattern, query_content_start):
+                        return True
+                continue
+
+            # Check if we're still in the query field
+            if in_query_field:
+                # If we hit another top-level property, we've exited the query field
+                if re.match(r"^\w+\s*:", line):
+                    in_query_field = False
+                    continue
+
+                # We're still in the query field, check for variable
+                if re.search(variable_pattern, line):
+                    return True
+
+        return False
+
     # Implemented to get correct type hints
     def __iter__(self) -> Iterator[BuildVariable]:
         return super().__iter__()

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_cdf_tk/data_classes/_built_resources.py

@@ -158,7 +158,7 @@ class BuiltResourceFull(BuiltResource[T_ID]):
     def load_resource_dict(
         self, environment_variables: dict[str, str | None], validate: bool = False
     ) -> dict[str, Any]:
-        content = self.build_variables.replace(safe_read(self.source.path))
+        content = self.build_variables.replace(safe_read(self.source.path), self.source.path)
         loader = cast(ResourceCRUD, get_crud(self.resource_dir, self.kind))
         raw = load_yaml_inject_variables(
             content,

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_cdf_tk/resource_classes/agent.py

@@ -55,3 +55,4 @@ class AgentYAML(ToolkitResource):
         "azure/gpt-4o-mini", description="The name of the model to use. Defaults to your CDF project's default model."
     )
     tools: list[AgentTool] | None = Field(None, description="A list of tools available to the agent.", max_length=20)
+    runtime_version: str | None = Field(None, description="The runtime version")

cognite_toolkit-0.6.90/cognite_toolkit/_cdf_tk/resource_classes/infield_cdmv1.py

@@ -0,0 +1,92 @@
+from typing import Any
+
+from .base import BaseModelResource, ToolkitResource
+
+
+class ObservationFeatureToggles(BaseModelResource):
+    """Feature toggles for observations."""
+
+    is_enabled: bool | None = None
+    is_write_back_enabled: bool | None = None
+    notifications_endpoint_external_id: str | None = None
+    attachments_endpoint_external_id: str | None = None
+
+
+class FeatureToggles(BaseModelResource):
+    """Feature toggles for InField location configuration."""
+
+    three_d: bool | None = None
+    trends: bool | None = None
+    documents: bool | None = None
+    workorders: bool | None = None
+    notifications: bool | None = None
+    media: bool | None = None
+    template_checklist_flow: bool | None = None
+    workorder_checklist_flow: bool | None = None
+    observations: ObservationFeatureToggles | None = None
+
+
+class AccessManagement(BaseModelResource):
+    """Access management configuration."""
+
+    template_admins: list[str] | None = None  # list of CDF group external IDs
+    checklist_admins: list[str] | None = None  # list of CDF group external IDs
+
+
+class ResourceFilters(BaseModelResource):
+    """Resource filters."""
+
+    spaces: list[str] | None = None
+
+
+class RootLocationDataFilters(BaseModelResource):
+    """Data filters for root location."""
+
+    general: ResourceFilters | None = None
+    assets: ResourceFilters | None = None
+    files: ResourceFilters | None = None
+    timeseries: ResourceFilters | None = None
+
+
+class DataExplorationConfig(BaseModelResource):
+    """Properties for DataExplorationConfig node.
+
+    Contains configuration for data exploration features:
+    - observations: Observations feature configuration
+    - activities: Activities configuration
+    - documents: Document configuration
+    - notifications: Notifications configuration
+    - assets: Asset page configuration
+    """
+
+    external_id: str
+
+    observations: dict[str, Any] | None = None  # ObservationsConfigFeature
+    activities: dict[str, Any] | None = None  # ActivitiesConfiguration
+    documents: dict[str, Any] | None = None  # DocumentConfiguration
+    notifications: dict[str, Any] | None = None  # NotificationsConfiguration
+    assets: dict[str, Any] | None = None  # AssetPageConfiguration
+
+
+class InfieldLocationConfigYAML(ToolkitResource):
+    """Properties for InFieldLocationConfig node.
+
+    Currently migrated fields:
+    - root_location_external_id: Reference to the LocationFilterDTO external ID
+    - feature_toggles: Feature toggles migrated from old configuration
+    - rootAsset: Direct relation to the root asset (space and externalId)
+    - app_instance_space: Application instance space from appDataInstanceSpace
+    - access_management: Template and checklist admin groups (from templateAdmins and checklistAdmins)
+    - disciplines: List of disciplines (from disciplines in FeatureConfiguration)
+    - data_filters: Data filters for general, assets, files, and timeseries (from dataFilters in old configuration)
+    - data_exploration_config: Direct relation to the DataExplorationConfig node (shared across all locations)
+    """
+
+    external_id: str
+
+    root_location_external_id: str | None = None
+    feature_toggles: FeatureToggles | None = None
+    app_instance_space: str | None = None
+    access_management: AccessManagement | None = None
+    data_filters: RootLocationDataFilters | None = None
+    data_exploration_config: DataExplorationConfig | None = None

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.89
+      image: cognite/toolkit:0.6.90
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.89
+      image: cognite/toolkit:0.6.90
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.89"
+version = "0.6.90"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.90/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.90"

cognite_toolkit-0.6.90/last_changelog_entry.md

@@ -0,0 +1,10 @@
+## cdf 
+
+### Fixed
+
+- List variables are converted to the correct SQL set format 
+
+
+## templates
+
+No changes.

cognite_toolkit-0.6.90/last_git_message.txt

@@ -0,0 +1,55 @@
+[CDF-26214] Correct list replacement in SQL (#2176)
+
+# Description
+
+Reviewer: note that since we want to implement the same fix in inline
+SQL in Transformation.sql files, I had to refactor a bit to send in the
+full path, and also add a bit of logic to identify the query property
+and get the right replacement. The resulting code is quite complex. I
+wonder if we should explore revisiting the replacement logic altogether.
+Maybe Pydantic custom serializers?
+
+
+Given a sql file _or_ a query property that has this:
+
+```wells AS (
+  SELECT
+    a.externalId AS asset_ext_id,
+  	a.top_level_asset AS top_level_asset
+  FROM cdf_nodes('{{ apm_source_data_space }}',
+                 'APM_Asset',
+                 '{{ apm_source_data_version }}') a
+  WHERE a.type = 'Well'
+    AND a.space IN {{ instance_spaces }}
+),
+```
+
+and a config.<env>.yaml like this: 
+
+```
+instance_spaces:
+    - sp_asset_a
+    - sp_asset_b
+```
+ 
+Before, the Toolkit replaced the placeholder to `["sp_asset_a",
+"sp_asset_b"]` which is not correct SQL. Now it will convert it into
+`("sp_asset_a", "sp_asset_b")` which is correct SQL.
+
+
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Fixed
+
+- List variables are converted to the correct SQL set format 
+
+
+## templates
+
+No changes.

cognite_toolkit-0.6.90/last_version.txt

@@ -0,0 +1 @@
+0.6.89

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.89"
+version = "0.6.90"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/tests/test_integration/test_commands/test_deploy.py

@@ -184,7 +184,12 @@ def get_changed_source_files(
             _, to_write = cmd._get_to_write(local_resource_by_id, cdf_resource_by_id, file_results, loader)
 
             new_content, extra_files = cmd._to_write_content(
-                original_content, to_write, resources, environment_variables, loader
+                source=original_content,
+                to_write=to_write,
+                resources=resources,
+                environment_variables=environment_variables,
+                loader=loader,
+                source_file=source_file,
             )
             new_content = remove_trailing_newline(new_content)
             if new_content != original_content:

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/tests/test_unit/test_cdf_tk/test_commands/test_pull.py

@@ -516,6 +516,7 @@ class TestPullCommand:
             resources=resources,
             environment_variables={},
             loader=DataSetsCRUD.create_loader(toolkit_client_approval.mock_client),
+            source_file=Path("my.Dataset.yaml"),
         )
         assert not extra_files, "This tests does not support testing extra files"
         assert actual.splitlines() == expected.splitlines()

{cognite_toolkit-0.6.89 → cognite_toolkit-0.6.90}/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py

@@ -69,10 +69,25 @@ suffix_text: {{ my_suffix_text }}
             selected_modules=set(),
         )
 
-        result = variables.replace(source_yaml, file_suffix=".sql")
+        result = variables.replace(source_yaml, Path("test.sql"))
 
         assert result == "dataset_id('ds_external_id')"
 
+    def test_replace_sql_list(self) -> None:
+        """Test that lists with mixed types in SQL files are formatted correctly."""
+        source_sql = """SELECT * FROM table WHERE column IN {{ my_list }}"""
+        variables = BuildVariables.load_raw(
+            {
+                "my_list": ["A", 123, None, True],
+            },
+            available_modules=set(),
+            selected_modules=set(),
+        )
+
+        result = variables.replace(source_sql, Path("test.sql"))
+
+        assert result == "SELECT * FROM table WHERE column IN ('A', 123, NULL, True)"
+
     def test_replace_inline_sql_preserve_double_quotes(self) -> None:
         source_yaml = """externalId: some_id
 name: Some Transformation
@@ -95,12 +110,76 @@ query: >-
             selected_modules=set(),
         )
 
-        result = variables.replace(source_yaml, file_suffix=".yaml")
+        result = variables.replace(source_yaml, Path("test.Transformation.yaml"))
 
         loaded = yaml.safe_load(result)
 
         assert loaded["query"] == 'select "fpso_uny" as externalId, "UNY" as uid, "UNY" as description'
 
+    def test_replace_yaml_query_field_list(self) -> None:
+        """Test that lists in multiline query fields are converted to SQL-style tuples."""
+        source_yaml = """externalId: some_id
+name: Some Transformation
+query: |
+  SELECT * FROM table
+  WHERE column IN {{ my_list }}
+  AND other_column = 'value'
+"""
+        variables = BuildVariables.load_raw(
+            {
+                "my_list": ["X", "Y", "Z"],
+            },
+            available_modules=set(),
+            selected_modules=set(),
+        )
+
+        result = variables.replace(source_yaml, Path("test.Transformation.yaml"))
+
+        loaded = yaml.safe_load(result)
+        assert "('X', 'Y', 'Z')" in loaded["query"]
+
+    def test_replace_yaml_query_field_list_mixed_types(self) -> None:
+        """Test that lists with mixed types in query fields are formatted correctly."""
+        source_yaml = """externalId: some_id
+name: Some Transformation
+query: >-
+  SELECT * FROM table WHERE column IN {{ my_list }}
+"""
+        variables = BuildVariables.load_raw(
+            {
+                "my_list": ["A", 123, None, True],
+            },
+            available_modules=set(),
+            selected_modules=set(),
+        )
+
+        result = variables.replace(source_yaml, Path("test.Transformation.yaml"))
+
+        loaded = yaml.safe_load(result)
+        assert loaded["query"] == "SELECT * FROM table WHERE column IN ('A', 123, NULL, True)"
+
+    def test_replace_yaml_non_query_field_preserves_list(self) -> None:
+        """Test that lists outside query fields still use YAML list format."""
+        source_yaml = """externalId: some_id
+name: Some Transformation
+tags: {{ my_list }}
+query: >-
+  SELECT * FROM table
+"""
+        variables = BuildVariables.load_raw(
+            {
+                "my_list": ["tag1", "tag2"],
+            },
+            available_modules=set(),
+            selected_modules=set(),
+        )
+
+        result = variables.replace(source_yaml)
+
+        loaded = yaml.safe_load(result)
+        # Tags should be a YAML list, not SQL tuple
+        assert loaded["tags"] == ["tag1", "tag2"]
+
     def test_get_module_variables_variable_preference_order(self) -> None:
         source_yaml = """
 modules: