cognite-toolkit 0.6.109__py3-none-any.whl → 0.6.110__py3-none-any.whl

cognite_toolkit/_cdf_tk/client/data_classes/charts_data.py

@@ -135,6 +135,26 @@ class ChartCoreTimeseries(BaseChartElement):
     view_reference: ViewId | None = None
     display_mode: str | None = None
 
+    def dump(self, camel_case: bool = True) -> dict[str, Any]:
+        data = super().dump(camel_case=camel_case)
+        if self.node_reference:
+            key = "nodeReference" if camel_case else "node_reference"
+            data[key] = self.node_reference.dump(include_instance_type=False)
+        if self.view_reference:
+            key = "viewReference" if camel_case else "view_reference"
+            data[key] = self.view_reference.dump(include_type=False)
+        return data
+
+    @classmethod
+    def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
+        """Load a ChartCoreTimeseries object from a dictionary."""
+        instance = super()._load(resource, cognite_client=cognite_client)
+        if "nodeReference" in resource:
+            instance.node_reference = NodeId.load(resource["nodeReference"])
+        if "viewReference" in resource:
+            instance.view_reference = ViewId.load(resource["viewReference"])
+        return instance
+
 
 @dataclass
 class ChartTimeseries(BaseChartElement):

cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
-from typing import Generic
+from typing import Generic, cast
+from uuid import uuid4
 
 from cognite.client.data_classes._base import (
     T_CogniteResource,
@@ -9,20 +10,24 @@ from cognite.client.data_classes.data_modeling import (
     EdgeApply,
     InstanceApply,
     NodeApply,
+    NodeId,
     View,
     ViewId,
 )
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartWrite
+from cognite_toolkit._cdf_tk.client.data_classes.charts_data import ChartCoreTimeseries, ChartSource, ChartTimeseries
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
 from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
 from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
-from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
+from cognite_toolkit._cdf_tk.commands._migrate.issues import ChartMigrationIssue, ConversionIssue, MigrationIssue
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector, T_WriteCogniteResource
+from cognite_toolkit._cdf_tk.storageio.selectors import ChartSelector
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types import (
     T_AssetCentricResourceExtended,
@@ -120,3 +125,123 @@ class AssetCentricMapper(
         if mapping.instance_id.space == MISSING_INSTANCE_SPACE:
             conversion_issue.missing_instance_space = f"Missing instance space for dataset ID {mapping.data_set_id!r}"
         return instance, conversion_issue
+
+
+class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
+    def __init__(self, client: ToolkitClient) -> None:
+        self.client = client
+
+    def map(self, source: Sequence[Chart]) -> Sequence[tuple[ChartWrite | None, MigrationIssue]]:
+        self._populate_cache(source)
+        output: list[tuple[ChartWrite | None, MigrationIssue]] = []
+        for item in source:
+            mapped_item, issue = self._map_single_item(item)
+            output.append((mapped_item, issue))
+        return output
+
+    def _populate_cache(self, source: Sequence[Chart]) -> None:
+        """Populate the internal cache with timeseries from the source charts.
+
+        Note that the consumption views are also cached as part of the timeseries lookup.
+        """
+        timeseries_ids: set[int] = set()
+        timeseries_external_ids: set[str] = set()
+        for chart in source:
+            for item in chart.data.time_series_collection or []:
+                if item.ts_id:
+                    timeseries_ids.add(item.ts_id)
+                if item.ts_external_id:
+                    timeseries_external_ids.add(item.ts_external_id)
+        if timeseries_ids:
+            self.client.migration.lookup.time_series(list(timeseries_ids))
+        if timeseries_external_ids:
+            self.client.migration.lookup.time_series(external_id=list(timeseries_external_ids))
+
+    def _map_single_item(self, item: Chart) -> tuple[ChartWrite | None, ChartMigrationIssue]:
+        issue = ChartMigrationIssue(chart_external_id=item.external_id)
+        time_series_collection = item.data.time_series_collection or []
+        timeseries_core_collection = self._create_timeseries_core_collection(time_series_collection, issue)
+        if issue.has_issues:
+            return None, issue
+
+        updated_source_collection = self._update_source_collection(
+            item.data.source_collection or [], time_series_collection, timeseries_core_collection
+        )
+
+        mapped_chart = item.as_write()
+        mapped_chart.data.core_timeseries_collection = timeseries_core_collection
+        mapped_chart.data.time_series_collection = None
+        mapped_chart.data.source_collection = updated_source_collection
+        return mapped_chart, issue
+
+    def _create_timeseries_core_collection(
+        self, time_series_collection: list[ChartTimeseries], issue: ChartMigrationIssue
+    ) -> list[ChartCoreTimeseries]:
+        timeseries_core_collection: list[ChartCoreTimeseries] = []
+        for ts_item in time_series_collection or []:
+            node_id, consumer_view_id = self._get_node_id_consumer_view_id(ts_item)
+
+            if node_id is None:
+                if ts_item.ts_id is not None:
+                    issue.missing_timeseries_ids.append(ts_item.ts_id)
+                elif ts_item.ts_external_id is not None:
+                    issue.missing_timeseries_external_ids.append(ts_item.ts_external_id)
+                else:
+                    issue.missing_timeseries_identifier.append(ts_item.id or "unknown")
+                continue
+
+            core_timeseries = self._create_new_timeseries_core(ts_item, node_id, consumer_view_id)
+            timeseries_core_collection.append(core_timeseries)
+        return timeseries_core_collection
+
+    def _create_new_timeseries_core(
+        self, ts_item: ChartTimeseries, node_id: NodeId, consumer_view_id: ViewId | None
+    ) -> ChartCoreTimeseries:
+        dumped = ts_item.dump(camel_case=True)
+        for asset_centric_key in ["tsId", "tsExternalId", "originalUnit"]:
+            dumped.pop(asset_centric_key, None)
+
+        dumped["nodeReference"] = node_id
+        dumped["viewReference"] = consumer_view_id
+        new_uuid = str(uuid4())
+        dumped["id"] = new_uuid
+        dumped["type"] = "coreTimeseries"
+        core_timeseries = ChartCoreTimeseries._load(dumped)
+        return core_timeseries
+
+    def _get_node_id_consumer_view_id(self, ts_item: ChartTimeseries) -> tuple[NodeId | None, ViewId | None]:
+        """Look up the node ID and consumer view ID for a given timeseries item.
+
+        Prioritizes lookup by internal ID, then by external ID.
+
+        Args:
+            ts_item: The ChartTimeseries item to look up.
+
+        Returns:
+            A tuple containing the consumer view ID and node ID, or None if not found.
+        """
+        node_id: NodeId | None = None
+        consumer_view_id: ViewId | None = None
+        for id_name, id_value in [("id", ts_item.ts_id), ("external_id", ts_item.ts_external_id)]:
+            if id_value is None:
+                continue
+            arg = {id_name: id_value}
+            node_id = self.client.migration.lookup.time_series(**arg)  # type: ignore[arg-type]
+            consumer_view_id = self.client.migration.lookup.time_series.consumer_view(**arg)  # type: ignore[arg-type]
+            if node_id is not None:
+                break
+        return node_id, consumer_view_id
+
+    def _update_source_collection(
+        self,
+        source_collection: list[ChartSource],
+        time_series_collection: list[ChartTimeseries],
+        timeseries_core_collection: list[ChartCoreTimeseries],
+    ) -> list[ChartSource]:
+        remove_ids = {ts_item.id for ts_item in time_series_collection if ts_item.id is not None}
+        updated_source_collection = [ts_item for ts_item in source_collection if ts_item.id not in remove_ids]
+        for core_ts_item in timeseries_core_collection:
+            # We cast there two as we set them in the _create_timeseries_core_collection method
+            new_source_item = ChartSource(id=cast(str, core_ts_item.id), type=cast(str, core_ts_item.type))
+            updated_source_collection.append(new_source_item)
+        return updated_source_collection

cognite_toolkit/_cdf_tk/commands/_migrate/issues.py

@@ -30,6 +30,27 @@ class MigrationIssue(MigrationObject):
         return True
 
 
+class ChartMigrationIssue(MigrationIssue):
+    """Represents a chart migration issue encountered during migration.
+
+    Attributes:
+        chart_external_id (str): The external ID of the chart that could not be migrated.
+    """
+
+    type: ClassVar[str] = "chartMigration"
+    chart_external_id: str
+    missing_timeseries_ids: list[int] = Field(default_factory=list)
+    missing_timeseries_external_ids: list[str] = Field(default_factory=list)
+    missing_timeseries_identifier: list[str] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this ChartMigrationIssue."""
+        return bool(
+            self.missing_timeseries_ids or self.missing_timeseries_external_ids or self.missing_timeseries_identifier
+        )
+
+
 class ReadIssue(MigrationIssue):
     """Represents a read issue encountered during migration."""
 

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.109
+      image: cognite/toolkit:0.6.110
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.109
+      image: cognite/toolkit:0.6.110
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.109"
+version = "0.6.110"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.109"
+__version__ = "0.6.110"

{cognite_toolkit-0.6.109.dist-info → cognite_toolkit-0.6.110.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.109
+Version: 0.6.110
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.109.dist-info → cognite_toolkit-0.6.110.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=0abeQr1Tfk4lkGaoXyrnFC28wDSlR_8UGrh10noGduQ,6085
-cognite_toolkit/_version.py,sha256=RN7f5cMrZyBLBF66iauaGYInNemTmy-2sr6H3Ysm8OU,24
+cognite_toolkit/_version.py,sha256=r8GW0BNwRXUAwN3axagFdsMsJonpDcvGzFglYAFiRio,24
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
 cognite_toolkit/_cdf_tk/constants.py,sha256=Gi7iGGzdUrOnBeIK6ix3XiBieHIwzLJO5BWjDI3a6l4,7082
@@ -77,7 +77,7 @@ cognite_toolkit/_cdf_tk/client/data_classes/base.py,sha256=J9Mhf7cYObYPWJ-ne98ec
 cognite_toolkit/_cdf_tk/client/data_classes/canvas.py,sha256=1yOSXkARz6QgIUnNryRy1UbV62dPAxE9qu5jfpOmPGE,48618
 cognite_toolkit/_cdf_tk/client/data_classes/capabilities.py,sha256=muqpAC2JLCFcEpRPzuh_3sS3o_q42WFyfsGzl-LfB_U,8773
 cognite_toolkit/_cdf_tk/client/data_classes/charts.py,sha256=l4LM1eqn8_2lRzw-DdwwL0zxgSczC0lp1LjgW-4wqPI,3770
-cognite_toolkit/_cdf_tk/client/data_classes/charts_data.py,sha256=nPbB-hQ7Z6D21CgBI3Zl8UQzsjlsWEuwCtBt93lYQ4Q,11242
+cognite_toolkit/_cdf_tk/client/data_classes/charts_data.py,sha256=Zcx3st3bMJE2aa5U6TepDqRzqf-LuTTWW5grSaVGr9E,12244
 cognite_toolkit/_cdf_tk/client/data_classes/extendable_cognite_file.py,sha256=0iyLiXEzB4WBU-DL6DZS6nD5E526cDsftMGEYXwI8r8,9764
 cognite_toolkit/_cdf_tk/client/data_classes/extended_filemetadata.py,sha256=8zfXl_bhkums3quJzdOwAjxVNY6B0hpAs6jbkekn79o,5488
 cognite_toolkit/_cdf_tk/client/data_classes/extended_filemetdata.py,sha256=gKA5UcDKweH7SlzXfyZCspMyHUo0t8R5DbzeCPpzInM,6002
@@ -130,10 +130,10 @@ cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=jNoqqq81lbdfDTAQ5w2c
 cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=Ew9JRYrd-Ol9G9csTzpnhXAgCFnX67MwDYOTsdJLP3E,16803
 cognite_toolkit/_cdf_tk/commands/_migrate/creators.py,sha256=FTu7w3G8KyPY8pagG3KdPpOmpLcjehaAg2auEy6iM7A,9605
 cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=_vMS_qAPj4yup1VnmmojPVigAZtyPQH7PM0Raby5tao,10619
-cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=Y7MrE6FGa15uvboBjNyWNlslsBv4FpeP5WsrFsooxsA,5678
+cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=Q4VZsOF7__xkbNsCbzzhu-1Q34kYrMks1sKUZCk6ItY,11731
 cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=i1eUsNX6Dueol9STIEwyksBnBsWUk13O8qHIjW964pM,7860
 cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py,sha256=ERn3qFrJFXdtXaMjHq3Gk7MxH03MGFk3FrtWCOBJQts,5544
-cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=lWSnuS3CfRDbA7i1g12gJ2reJnQcLmZWxHDK19-Wxkk,5772
+cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=L2-kODPavEwcuhte7EXANK2-rH7reiq-uNqr-3ub-no,6575
 cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py,sha256=wrdBH5P6NgiZQSYLR0iJ3ZvqfQ5fY-_Ne2yKv9E1g4o,16277
 cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py,sha256=RfqaNoso5CyBwc-p6ckwcYqBfZXKhdJgdGIyd0TATaI,2635
 cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py,sha256=N1H_-rBpPUD6pbrlcofn1uEK1bA694EUXEe1zIXeqyo,2489
@@ -300,13 +300,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=nravoEwNPHKAQxpBkVmYEWs32zKaBJF4A0A6qGEqYYs,668
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=2ds1UMLZ_3DtSe-sdlKQkyKH1cMvFlJczJmuHPxrTK8,2431
-cognite_toolkit/_resources/cdf.toml,sha256=1HiibQI4-kcxjk1K2VnvDNilg8K6H6EOpdhTYK8od6w,488
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=sajGLbABAzcw49Ppp9445f-X0UR7aaEHM8zyQqa8OK4,668
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=FN7MyRwPdPA03R2VccxMZiKv58wrE1zxTUyx-oaPs2M,2431
+cognite_toolkit/_resources/cdf.toml,sha256=fr1qQg8VjnRir6nN2fiK6dBQWp2GiKPKON-kM9XYktM,488
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.109.dist-info/METADATA,sha256=eDd46Parkn2Cs5ujEbVM9lTocNMA3GZh-IGSOyRl9zM,4502
-cognite_toolkit-0.6.109.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.109.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.109.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.109.dist-info/RECORD,,
+cognite_toolkit-0.6.110.dist-info/METADATA,sha256=3NOWs5ZjtTei1p98UMZs-9QKy9clBW0aqa1rY0hIp_U,4502
+cognite_toolkit-0.6.110.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.110.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.110.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.110.dist-info/RECORD,,