cognite-toolkit 0.6.102__tar.gz → 0.6.103__tar.gz

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/.github/workflows/demo.yml

@@ -19,7 +19,7 @@ jobs:
         LOGIN_FLOW: client_credentials
     name: Loadmaster
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
       - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/.github/workflows/release.yaml

@@ -18,7 +18,7 @@ jobs:
       IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }}
       IDP_TOKEN_URL: ${{ vars.IDP_TOKEN_URL }}
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
       - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
         with:
           python-version: 3.11
@@ -39,7 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: CD
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
         with:
           fetch-depth: 0
       - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.102
+Version: 0.6.103
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cdf.toml

@@ -32,7 +32,7 @@ dev = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.102"
+version = "0.6.103"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_cdf_tk/apps/_download_app.py

@@ -10,6 +10,7 @@ from cognite_toolkit._cdf_tk.commands import DownloadCommand
 from cognite_toolkit._cdf_tk.constants import DATA_DEFAULT_DIR
 from cognite_toolkit._cdf_tk.storageio import (
     AssetIO,
+    ChartIO,
     HierarchyIO,
     InstanceIO,
     RawIO,
@@ -17,6 +18,8 @@ from cognite_toolkit._cdf_tk.storageio import (
 from cognite_toolkit._cdf_tk.storageio.selectors import (
     AssetCentricSelector,
     AssetSubtreeSelector,
+    ChartExternalIdSelector,
+    ChartSelector,
     DataSetSelector,
     InstanceSpaceSelector,
     RawTableSelector,
@@ -27,6 +30,7 @@ from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
     AssetInteractiveSelect,
     DataModelingSelect,
+    InteractiveChartSelect,
     RawTableInteractiveSelect,
 )
 
@@ -50,6 +54,10 @@ class InstanceFormats(str, Enum):
     ndjson = "ndjson"
 
 
+class ChartFormats(str, Enum):
+    ndjson = "ndjson"
+
+
 class InstanceTypes(str, Enum):
     node = "node"
     edge = "edge"
@@ -71,6 +79,7 @@ class DownloadApp(typer.Typer):
         self.command("assets")(self.download_assets_cmd)
         self.command("hierarchy")(self.download_hierarchy_cmd)
         self.command("instances")(self.download_instances_cmd)
+        self.command("charts")(self.download_charts_cmd)
 
     @staticmethod
     def download_main(ctx: typer.Context) -> None:
@@ -476,3 +485,76 @@ class DownloadApp(typer.Typer):
                 verbose=verbose,
             )
         )
+
+    @staticmethod
+    def download_charts_cmd(
+        ctx: typer.Context,
+        external_ids: Annotated[
+            list[str] | None,
+            typer.Argument(
+                help="List of chart external IDs to download. If not provided, an interactive selection will be made.",
+            ),
+        ] = None,
+        file_format: Annotated[
+            ChartFormats,
+            typer.Option(
+                "--format",
+                "-f",
+                help="Format for downloading the charts.",
+            ),
+        ] = ChartFormats.ndjson,
+        compression: Annotated[
+            CompressionFormat,
+            typer.Option(
+                "--compression",
+                "-z",
+                help="Compression format to use when downloading the instances.",
+            ),
+        ] = CompressionFormat.none,
+        output_dir: Annotated[
+            Path,
+            typer.Option(
+                "--output-dir",
+                "-o",
+                help="Where to download the charts.",
+                allow_dash=True,
+            ),
+        ] = DEFAULT_DOWNLOAD_DIR,
+        limit: Annotated[
+            int,
+            typer.Option(
+                "--limit",
+                "-l",
+                help="The maximum number of charts to download. Use -1 to download all charts.",
+            ),
+        ] = 1000,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """This command will download Charts from CDF into a temporary directory."""
+        cmd = DownloadCommand()
+        client = EnvironmentVariables.create_from_environment().get_client()
+        selector: ChartSelector
+        if external_ids is None:
+            selected_external_ids = InteractiveChartSelect(client).select_external_ids()
+            selector = ChartExternalIdSelector(external_ids=tuple(selected_external_ids))
+        else:
+            selector = ChartExternalIdSelector(external_ids=tuple(external_ids))
+
+        cmd.run(
+            lambda: cmd.download(
+                selectors=[selector],
+                io=ChartIO(client),
+                output_dir=output_dir,
+                file_format=f".{file_format.value}",
+                compression=compression.value,
+                limit=limit if limit != -1 else None,
+                verbose=verbose,
+            )
+        )

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -10,7 +10,7 @@ from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
 from ._base import Page, UploadableStorageIO
-from .selectors import AllChartsSelector, CanvasSelector, ChartOwnerSelector, ChartSelector
+from .selectors import AllChartsSelector, CanvasSelector, ChartExternalIdSelector, ChartOwnerSelector, ChartSelector
 
 
 class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
@@ -30,29 +30,15 @@ class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
             ...
         elif isinstance(selector, ChartOwnerSelector):
             selected_charts = ChartList([chart for chart in selected_charts if chart.owner_id == selector.owner_id])
+        elif isinstance(selector, ChartExternalIdSelector):
+            external_id_set = set(selector.external_ids)
+            selected_charts = ChartList([chart for chart in selected_charts if chart.external_id in external_id_set])
         else:
             raise ToolkitNotImplementedError(f"Unsupported selector type {type(selector).__name__!r} for ChartIO")
 
         if limit is not None:
             selected_charts = ChartList(selected_charts[:limit])
         for chunk in chunker_sequence(selected_charts, self.CHUNK_SIZE):
-            ts_ids_to_lookup = {
-                ts_ref.ts_id
-                for chart in chunk
-                for ts_ref in chart.data.time_series_collection or []
-                if ts_ref.ts_external_id is None and ts_ref.ts_id is not None
-            }
-
-            if ts_ids_to_lookup:
-                retrieved_ts = self.client.time_series.retrieve_multiple(
-                    ids=list(ts_ids_to_lookup), ignore_unknown_ids=True
-                )
-                id_to_external_id = {ts.id: ts.external_id for ts in retrieved_ts}
-
-                for chart in chunk:
-                    for ts_ref in chart.data.time_series_collection or []:
-                        if ts_ref.ts_id in id_to_external_id:
-                            ts_ref.ts_external_id = id_to_external_id[ts_ref.ts_id]
             yield Page(worker_id="main", items=chunk)
 
     def count(self, selector: ChartSelector) -> int | None:
@@ -62,7 +48,32 @@ class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
     def data_to_json_chunk(
         self, data_chunk: Sequence[Chart], selector: ChartSelector | None = None
     ) -> list[dict[str, JsonVal]]:
-        return [chart.as_write().dump() for chart in data_chunk]
+        self._populate_timeseries_id_cache(data_chunk)
+        return [self._dump_resource(chart) for chart in data_chunk]
+
+    def _populate_timeseries_id_cache(self, data_chunk: Sequence[Chart]) -> None:
+        timeseries_ids: set[int] = set()
+        for chart in data_chunk:
+            for item in chart.data.time_series_collection or []:
+                if item.ts_id is not None and item.ts_external_id is None:
+                    # We only look-up the internalID if the externalId is missing
+                    timeseries_ids.add(item.ts_id)
+        if timeseries_ids:
+            self.client.lookup.time_series.external_id(list(timeseries_ids))
+
+    def _dump_resource(self, chart: Chart) -> dict[str, JsonVal]:
+        dumped = chart.as_write().dump()
+        if isinstance(data := dumped.get("data"), dict) and isinstance(
+            collection := data.get("timeSeriesCollection"), list
+        ):
+            for item in collection:
+                ts_id = item.pop("tsId", None)
+                if ts_id and item.get("tsExternalId") is None:
+                    # We only look-up the externalID if it is missing
+                    ts_external_id = self.client.lookup.time_series.external_id(ts_id)
+                    if ts_external_id is not None:
+                        item["tsExternalId"] = ts_external_id
+        return dumped
 
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> ChartWrite:
         return ChartWrite._load(item_json)

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py

@@ -5,7 +5,7 @@ from pydantic import Field, TypeAdapter
 from ._asset_centric import AssetCentricFileSelector, AssetCentricSelector, AssetSubtreeSelector, DataSetSelector
 from ._base import DataSelector
 from ._canvas import CanvasSelector
-from ._charts import AllChartsSelector, ChartOwnerSelector, ChartSelector
+from ._charts import AllChartsSelector, ChartExternalIdSelector, ChartOwnerSelector, ChartSelector
 from ._datapoints import (
     DataPointsFileSelector,
     ExternalIdColumn,
@@ -32,7 +32,8 @@ Selector = Annotated[
     | AssetSubtreeSelector
     | AssetCentricFileSelector
     | DataSetSelector
-    | DataPointsFileSelector,
+    | DataPointsFileSelector
+    | ChartExternalIdSelector,
     Field(discriminator="type"),
 ]
 
@@ -45,6 +46,7 @@ __all__ = [
     "AssetCentricSelector",
     "AssetSubtreeSelector",
     "CanvasSelector",
+    "ChartExternalIdSelector",
     "ChartOwnerSelector",
     "ChartSelector",
     "DataPointsFileSelector",

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py

@@ -1,3 +1,4 @@
+import hashlib
 from abc import ABC
 from typing import Literal
 
@@ -29,3 +30,16 @@ class AllChartsSelector(ChartSelector):
 
     def __str__(self) -> str:
         return "all"
+
+
+class ChartExternalIdSelector(ChartSelector):
+    type: Literal["chartExternalId"] = "chartExternalId"
+    external_ids: tuple[str, ...]
+
+    @property
+    def group(self) -> str:
+        return "Charts"
+
+    def __str__(self) -> str:
+        hash_ = hashlib.md5(",".join(sorted(self.external_ids)).encode()).hexdigest()[:8]
+        return f"chart_count_{len(self.external_ids)}_hash_{hash_}"

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_cdf_tk/utils/interactive_select.py

@@ -14,9 +14,6 @@ from cognite.client.data_classes import (
     filters,
 )
 from cognite.client.data_classes.aggregations import Count
-from cognite.client.data_classes.capabilities import (
-    UserProfilesAcl,
-)
 from cognite.client.data_classes.data_modeling import ContainerId, NodeList, Space, SpaceList, View, ViewId, ViewList
 from cognite.client.data_classes.data_modeling.statistics import SpaceStatistics
 from cognite.client.utils import ms_to_datetime
@@ -28,7 +25,7 @@ from cognite_toolkit._cdf_tk.client.data_classes.canvas import Canvas
 from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartList, Visibility
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
 from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
-from cognite_toolkit._cdf_tk.exceptions import AuthorizationError, ToolkitMissingResourceError, ToolkitValueError
+from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError, ToolkitValueError
 
 from . import humanize_collection
 from .aggregators import (
@@ -406,13 +403,6 @@ class InteractiveChartSelect:
         return user_response
 
     def _select_external_ids(self, select_filter: ChartFilter) -> list[str]:
-        if not self.client.iam.verify_capabilities(
-            UserProfilesAcl([UserProfilesAcl.Action.Read], scope=UserProfilesAcl.Scope.All())
-        ):
-            raise AuthorizationError(
-                "The current user does not have permission to list user profiles, "
-                "which is required to select Charts owned by a specific user."
-            )
         available_charts = self.client.charts.list(visibility=(select_filter.visibility or "PUBLIC"))
         if select_filter.select_all and select_filter.owned_by is None:
             return [chart.external_id for chart in available_charts]

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.102
+      image: cognite/toolkit:0.6.103
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.102
+      image: cognite/toolkit:0.6.103
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.102"
+version = "0.6.103"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.103/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.103"

cognite_toolkit-0.6.103/last_changelog_entry.md

@@ -0,0 +1,9 @@
+## cdf 
+
+### Added
+
+- [alpha] `cdf data download charts` 
+
+## templates
+
+No changes.

cognite_toolkit-0.6.103/last_git_message.txt

@@ -0,0 +1,21 @@
+[CDF-24842] 📈Charts download (#2209)
+
+# Description
+
+Command for downloading chars. Turns out most of the logic was already
+there.
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Added
+
+- [alpha] `cdf data download charts` 
+
+## templates
+
+No changes.

cognite_toolkit-0.6.103/last_version.txt

@@ -0,0 +1 @@
+0.6.102

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.102"
+version = "0.6.103"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/tests/test_unit/test_cdf_tk/test_storageio/test_selectors.py

@@ -19,6 +19,7 @@ from cognite_toolkit._cdf_tk.storageio.selectors import (
     AllChartsSelector,
     AssetCentricFileSelector,
     AssetSubtreeSelector,
+    ChartExternalIdSelector,
     ChartOwnerSelector,
     DataPointsFileSelector,
     DataSelector,
@@ -137,6 +138,17 @@ def example_selector_data() -> Iterable[tuple]:
         DatapointsIO.KIND,
         id="DataPointsFileSelector",
     )
+    yield pytest.param(
+        {
+            "type": "chartExternalId",
+            "kind": "Charts",
+            "externalIds": ["5400ab34-37f1-470d-b965-d81404ac92d8"],
+        },
+        ChartExternalIdSelector,
+        ChartIO,
+        ChartIO.KIND,
+        id="ChartExternalIdSelector",
+    )
 
 
 @pytest.fixture(scope="module")

{cognite_toolkit-0.6.102 → cognite_toolkit-0.6.103}/uv.lock

@@ -272,7 +272,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.6.102"
+version = "0.6.103"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.6.102/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.6.102"

cognite_toolkit-0.6.102/last_changelog_entry.md

@@ -1,9 +0,0 @@
-## cdf 
-
-### Improved
-
-- Interactive onboarding in cdf init
-
-## templates
-
-No changes.

cognite_toolkit-0.6.102/last_git_message.txt

@@ -1,41 +0,0 @@
-[CDF-26176] interactive init (#2172)
-
-# Description
-
-The `cdf init` was a missed opportunity to onboard users. Let's provide
-an interactive experience in v0.7!
-
-
-https://github.com/user-attachments/assets/b5c6da54-57b2-487f-a55e-1ddd65d9cd95
-
-How to test: 
-
-in a fresh directory, run 
-
-```
-pip install --upgrade git+https://github.com/cognitedata/toolkit.git@better-init
-cdf init --seven 
-```
-
-(`--seven ` is a hidden flag to enable the feature since we don't have a
-cdf.toml to get flags from at this stage)
-
-## Changelog
-
-- [x] Patch
-- [ ] Skip
-
-## cdf
-
-### Improved
-
-- Interactive onboarding in cdf init
-
-## templates
-
-No changes.
-
----------
-
-Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
-Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com>

cognite_toolkit-0.6.102/last_version.txt

@@ -1 +0,0 @@
-0.6.101