cognite-toolkit 0.5.63__tar.gz → 0.5.64__tar.gz

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.5.63
+Version: 0.5.64
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/cdf.toml

@@ -25,4 +25,4 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.5.63"
+version = "0.5.64"

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/cognite_toolkit/_builtin_modules/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.5.63"
+version = "0.5.64"
 
 
 [plugins]

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py

@@ -46,6 +46,7 @@ from cognite_toolkit._cdf_tk.utils import (
     calculate_secure_hash,
 )
 from cognite_toolkit._cdf_tk.utils.cdf import read_auth, try_find_error
+from cognite_toolkit._cdf_tk.utils.text import suffix_description
 
 from .auth_loaders import GroupAllScopedLoader
 from .data_organization_loaders import DataSetsLoader
@@ -464,20 +465,15 @@ class FunctionScheduleLoader(
             )
             self.authentication_by_id[identifier] = credentials
             auth_hash = calculate_secure_hash(credentials.dump(camel_case=True), shorten=True)
-            extra_str = f" {self._hash_key}: {auth_hash}"
-            if "description" not in resource:
-                resource["description"] = extra_str[1:]
-            elif resource["description"].endswith(extra_str[1:]):
-                # The hash is already in the description
-                ...
-            elif len(resource["description"]) + len(extra_str) < self._description_character_limit:
-                resource["description"] += f"{extra_str}"
-            else:
-                LowSeverityWarning(f"Description is too long for schedule {identifier!r}. Truncating...").print_warning(
-                    console=self.console
-                )
-                truncation = self._description_character_limit - len(extra_str) - 3
-                resource["description"] = f"{resource['description'][:truncation]}...{extra_str}"
+            extra_str = f"{self._hash_key}: {auth_hash}"
+            resource["description"] = suffix_description(
+                extra_str,
+                resource.get("description"),
+                self._description_character_limit,
+                self.get_id(resource),
+                self.display_name,
+                self.console,
+            )
         return resources
 
     def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> FunctionScheduleWrite:

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py

@@ -1,5 +1,7 @@
+import json
 from collections.abc import Hashable, Iterable, Sequence
 from functools import lru_cache
+from pathlib import Path
 from typing import Any, cast, final
 
 from cognite.client.data_classes import (
@@ -28,7 +30,9 @@ from cognite_toolkit._cdf_tk.exceptions import (
 )
 from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceContainerLoader, ResourceLoader
 from cognite_toolkit._cdf_tk.resource_classes import TimeSeriesYAML
+from cognite_toolkit._cdf_tk.utils import calculate_hash
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable, dm_identifier
+from cognite_toolkit._cdf_tk.utils.text import suffix_description
 
 from .auth_loaders import GroupAllScopedLoader, SecurityCategoryLoader
 from .classic_loaders import AssetLoader
@@ -216,6 +220,9 @@ class DatapointSubscriptionLoader(
         }
     )
 
+    _hash_key = "cdf-hash"
+    _description_character_limit = 1000
+
     @property
     def display_name(self) -> str:
         return "timeseries subscriptions"
@@ -325,25 +332,58 @@ class DatapointSubscriptionLoader(
     ) -> Iterable[DatapointSubscription]:
         return iter(self.client.time_series.subscriptions)
 
+    def load_resource_file(
+        self, filepath: Path, environment_variables: dict[str, str | None] | None = None
+    ) -> list[dict[str, Any]]:
+        resources = super().load_resource_file(filepath, environment_variables)
+        for resource in resources:
+            if "timeSeriesIds" not in resource and "instanceIds" not in resource:
+                continue
+            # If the timeSeriesIds or instanceIds is set, we need to add the auth hash to the description.
+            # such that we can detect if the subscription has changed.
+            content: dict[str, object] = {}
+            if "timeSeriesIds" in resource:
+                content["timeSeriesIds"] = resource["timeSeriesIds"]
+            if "instanceIds" in resource:
+                content["instanceIds"] = resource["instanceIds"]
+            timeseries_hash = calculate_hash(json.dumps(content), shorten=True)
+            extra_str = f"{self._hash_key}: {timeseries_hash}"
+            resource["description"] = suffix_description(
+                extra_str,
+                resource.get("description"),
+                self._description_character_limit,
+                self.get_id(resource),
+                self.display_name,
+                self.console,
+            )
+
+        return resources
+
     def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> DataPointSubscriptionWrite:
         if ds_external_id := resource.pop("dataSetExternalId", None):
             resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
         return DataPointSubscriptionWrite._load(resource)
 
     def dump_resource(self, resource: DatapointSubscription, local: dict[str, Any] | None = None) -> dict[str, Any]:
-        dumped = resource.as_write().dump()
+        if resource.filter is not None:
+            dumped = resource.as_write().dump()
+        else:
+            # If filter is not set, the subscription uses explicit timeSeriesIds, which are not returned in the
+            # response. Calling .as_write() in this case raises ValueError because either filter or
+            # timeSeriesIds must be set.
+            dumped = resource.dump()
+            for server_prop in ("createdTime", "lastUpdatedTime", "timeSeriesCount"):
+                dumped.pop(server_prop, None)
         local = local or {}
         if data_set_id := dumped.pop("dataSetId", None):
             dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
-        if "timeSeriesIds" not in dumped:
-            return dumped
-        # Sorting the timeSeriesIds in the local order
-        # Sorting in the same order as the local file.
-        ts_order_by_id = {ts_id: no for no, ts_id in enumerate(local.get("timeSeriesIds", []))}
-        end_of_list = len(ts_order_by_id)
-        dumped["timeSeriesIds"] = sorted(
-            dumped["timeSeriesIds"], key=lambda ts_id: ts_order_by_id.get(ts_id, end_of_list)
-        )
+        # timeSeriesIds and instanceIds are not returned in the response, so we need to add them
+        # to the dumped resource if they are set in the local resource. If there is a discrepancy between
+        # the local and dumped resource, th hash added to the description will change.
+        if "timeSeriesIds" in local:
+            dumped["timeSeriesIds"] = local["timeSeriesIds"]
+        if "instanceIds" in local:
+            dumped["instanceIds"] = local["instanceIds"]
         return dumped
 
     def diff_list(

cognite_toolkit-0.5.64/cognite_toolkit/_cdf_tk/utils/text.py

@@ -0,0 +1,42 @@
+from collections.abc import Hashable
+
+from rich.console import Console
+
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
+
+
+def suffix_description(
+    suffix: str,
+    description: str | None,
+    description_character_limit: int,
+    identifier: Hashable,
+    resource_type: str,
+    console: Console | None = None,
+) -> str:
+    """Appends a suffix to a description if it is not already present.
+    If the description is too long after appending the suffix, it will be truncated to fit within the character limit.
+
+    Args:
+        suffix: The suffix to append to the description.
+        description: The original description to which the suffix will be appended.
+        description_character_limit: The maximum number of characters allowed in the description after appending the suffix.
+        identifier: Hashable identifier for the resource, used in warnings.
+        resource_type:  Type of the resource, used in warnings.
+        console: Console object for printing warnings.
+
+    Returns:
+        str: The modified description with the suffix appended, or truncated if necessary.
+    """
+    if description is None or description == "":
+        return suffix
+    elif description.endswith(suffix):
+        # The suffix is already in the description
+        return description
+    elif len(description) + len(suffix) + 1 < description_character_limit:
+        return f"{description} {suffix}"
+    else:
+        LowSeverityWarning(f"Description is too long for {resource_type} {identifier!r}. Truncating...").print_warning(
+            console=console
+        )
+        truncation = description_character_limit - len(suffix) - 3
+        return f"{description[:truncation]}...{suffix}"

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.5.63
+      image: cognite/toolkit:0.5.64
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.5.63
+      image: cognite/toolkit:0.5.64
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit-0.5.64/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.5.64"

cognite_toolkit-0.5.64/last_changelog_entry.md

@@ -0,0 +1,11 @@
+## cdf 
+
+### Fixed
+
+- Running `cdf deploy` with a `DatapointSubscription` defined with
+timeseries IDs no longer raise a `ValueError` when the
+`DatapointSubscription` has already been deployed.
+
+## templates
+
+No changes.

cognite_toolkit-0.5.64/last_git_message.txt

@@ -0,0 +1,31 @@
+[CDF-25010] 🪨Datapoint subscription with timeSeriesIds or InstanceIds (#1659)
+
+# Description
+
+In Toolkit we assume that datapoint subscription resources defined with
+timeseries IDs, returns these timeseries IDs when retrieving. This is
+not the case and thus is causing the current bug.
+
+In addition, this means that if a datapoint subscription is defined with
+timeseries IDs or instance IDs, we will not be able to detect that the
+changes between the local and CDF version of the timeseries. To account
+for this, I add a hash to the description and can be used to detect
+changes.
+
+## Changelog
+
+- [x] Patch
+- [ ] Minor
+- [ ] Skip
+
+## cdf
+
+### Fixed
+
+- Running `cdf deploy` with a `DatapointSubscription` defined with
+timeseries IDs no longer raise a `ValueError` when the
+`DatapointSubscription` has already been deployed.
+
+## templates
+
+No changes.

cognite_toolkit-0.5.64/last_version.txt

@@ -0,0 +1 @@
+0.5.63

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.5.63"
+version = "0.5.64"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/tests/data/build_group_with_unknown_acl/_build_environment.yaml

@@ -4,4 +4,4 @@ project: pytest-project
 type: dev
 selected:
 - modules
-cdf_toolkit_version: 0.5.63
+cdf_toolkit_version: 0.5.64

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/tests/test_integration/conftest.py

@@ -9,6 +9,7 @@ from cognite.client.data_classes import (
     Asset,
     AssetList,
     AssetWrite,
+    DataSet,
     DataSetList,
     DataSetWrite,
     DataSetWriteList,
@@ -118,6 +119,18 @@ def toolkit_space(cognite_client: CogniteClient) -> Space:
     return cognite_client.data_modeling.spaces.apply(SpaceApply(space="toolkit_test_space"))
 
 
+@pytest.fixture(scope="session")
+def toolkit_dataset(cognite_client: CogniteClient) -> DataSet:
+    """Returns the dataset name used for toolkit tests."""
+    dataset = DataSetWrite(
+        external_id="toolkit_tests_dataset", name="Toolkit Test DataSet", description="Toolkit DataSet used in tests"
+    )
+    retrieved = cognite_client.data_sets.retrieve(external_id=dataset.external_id)
+    if retrieved is None:
+        return cognite_client.data_sets.create(dataset)
+    return retrieved
+
+
 @pytest.fixture
 def build_dir() -> Path:
     pidid = os.getpid()

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/tests/test_integration/test_loaders/test_resource_loaders.py

@@ -13,6 +13,7 @@ from cognite.client.data_classes import (
     AssetWriteList,
     ClientCredentials,
     DataPointSubscriptionWrite,
+    DataSet,
     Function,
     FunctionSchedule,
     FunctionSchedulesList,
@@ -21,7 +22,9 @@ from cognite.client.data_classes import (
     FunctionTaskParameters,
     GroupWrite,
     LabelDefinitionWrite,
+    TimeSeriesList,
     TimeSeriesWrite,
+    TimeSeriesWriteList,
     WorkflowVersionUpsert,
     WorkflowVersionUpsertList,
     filters,
@@ -180,6 +183,30 @@ class TestFunctionScheduleLoader:
                 toolkit_client.functions.schedules.delete(created[0].id)
 
 
+@pytest.fixture(scope="session")
+def three_timeseries(cognite_client: CogniteClient) -> TimeSeriesList:
+    ts_list = TimeSeriesWriteList(
+        [
+            TimeSeriesWrite(
+                external_id=f"toolkit_test_timeseries_{i}",
+                name=f"Toolkit Test TimeSeries {i}",
+                is_step=False,
+                is_string=False,
+            )
+            for i in range(1, 4)
+        ]
+    )
+    retrieved = cognite_client.time_series.retrieve_multiple(
+        external_ids=ts_list.as_external_ids(), ignore_unknown_ids=True
+    )
+    existing = set(retrieved.as_external_ids())
+    to_create = [ts for ts in ts_list if ts.external_id not in existing]
+    if to_create:
+        created = cognite_client.time_series.create(to_create)
+        retrieved.extend(created)
+    return retrieved
+
+
 class TestDatapointSubscriptionLoader:
     def test_delete_non_existing(self, cognite_client: CogniteClient) -> None:
         loader = DatapointSubscriptionLoader(cognite_client, None)
@@ -212,6 +239,40 @@ class TestDatapointSubscriptionLoader:
         finally:
             loader.delete([sub.external_id])
 
+    def test_no_redeploy_ids_defined(
+        self, toolkit_client: ToolkitClient, toolkit_dataset: DataSet, three_timeseries: TimeSeriesList
+    ) -> None:
+        definition_yaml = f"""externalId: toolkit_test_no_redeploy_ids_defined
+name: Test no redeploy IDs defined
+partitionCount: 1
+dataSetExternalId: {toolkit_dataset.external_id}
+timeSeriesIds:
+- {three_timeseries[0].external_id}
+- {three_timeseries[1].external_id}
+- {three_timeseries[2].external_id}
+"""
+        loader = DatapointSubscriptionLoader.create_loader(toolkit_client)
+
+        filepath = MagicMock(spec=Path)
+        filepath.read_text.return_value = definition_yaml
+
+        resource_dict = loader.load_resource_file(filepath, {})
+        assert len(resource_dict) == 1
+        resource = loader.load_resource(resource_dict[0])
+        assert isinstance(resource, DataPointSubscriptionWrite)
+        if not loader.retrieve([resource.external_id]):
+            _ = loader.create(DatapointSubscriptionWriteList([resource]))
+
+        worker = ResourceWorker(loader)
+        to_create, to_change, to_delete, unchanged = worker.prepare_resources([filepath])
+
+        assert {
+            "create": len(to_create),
+            "change": len(to_change),
+            "delete": len(to_delete),
+            "unchanged": len(unchanged),
+        } == {"create": 0, "change": 0, "delete": 0, "unchanged": 1}
+
 
 class TestLabelLoader:
     def test_delete_non_existing(self, cognite_client: CogniteClient) -> None:

cognite_toolkit-0.5.64/tests/test_unit/test_cdf_tk/test_utils/test_text.py

@@ -0,0 +1,34 @@
+from unittest.mock import MagicMock
+
+import pytest
+from rich.console import Console
+
+from cognite_toolkit._cdf_tk.utils.text import suffix_description
+
+
+class TestSuffixDescription:
+    @pytest.mark.parametrize(
+        "description, expected_description, expect_warning",
+        [
+            (None, "<suffix>", False),
+            ("", "<suffix>", False),
+            ("Existing description", "Existing description <suffix>", False),
+            ("Existing description <suffix>", "Existing description <suffix>", False),
+            (
+                "A very long description that exceeds the limit with more than one character",
+                "A very long descrip...<suffix>",
+                True,
+            ),
+        ],
+    )
+    def test_suffix_description(self, description: str | None, expected_description: str, expect_warning: True) -> None:
+        console = MagicMock(spec=Console)
+        limit = 30
+        actual = suffix_description("<suffix>", description, limit, "identifier", "my_resource_type", console=console)
+
+        assert actual == expected_description
+        assert len(actual) <= limit
+        if expect_warning:
+            assert console.print.call_count == 1
+            message = "".join(console.print.call_args[0])
+            assert "Description is too long for my_resource_type 'identifier'." in message

{cognite_toolkit-0.5.63 → cognite_toolkit-0.5.64}/uv.lock

@@ -192,7 +192,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.5.63"
+version = "0.5.64"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.5.63/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.5.63"

cognite_toolkit-0.5.63/last_changelog_entry.md

@@ -1,10 +0,0 @@
-## cdf 
-
-### Added
-
-- [alpha] `cdf profile transformations` profiling all transformations
-that writes to a given destination.
-
-## templates
-
-No changes.

cognite_toolkit-0.5.63/last_git_message.txt

@@ -1,30 +0,0 @@
-[CDF-24930] 🤖 Transformation profiler try2 (#1661)
-
-# Description
-
-Stacked on #1653 #1655 #1660 #1664 
-
-In the migration work, we need to have a simple way of understanding the
-data flow of a project. This profiling command gives a quick overview.
-
-
-![image](https://github.com/user-attachments/assets/6273dd7c-8f56-4077-8ee5-6d04d624cbe7)
-
-# Todo - tests
-
-## Changelog
-
-- [x] Patch
-- [ ] Minor
-- [ ] Skip
-
-## cdf
-
-### Added
-
-- [alpha] `cdf profile transformations` profiling all transformations
-that writes to a given destination.
-
-## templates
-
-No changes.

cognite_toolkit-0.5.63/last_version.txt

@@ -1 +0,0 @@
-0.5.62