cognite-toolkit 0.7.16__tar.gz → 0.7.18__tar.gz

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/.github/workflows/demo.yml

@@ -19,7 +19,7 @@ jobs:
         LOGIN_FLOW: client_credentials
     name: Loadmaster
     steps:
-      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
       - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
@@ -33,8 +33,6 @@ jobs:
         run: uv run cdf modules init demo_project --clean --all --library-url "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" --library-checksum "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"
       - name: "Pre-processing for demo environment"
         run: uv run python ./demo/preproc.py
-      - name: "Not collect metrics"
-        run: uv run cdf collect opt-out
       - name: "Build the templates"
         run: uv run cdf build --env demo -o ./demo_project
       - name: "Verify and create access rights"

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/.github/workflows/release.yaml

@@ -18,7 +18,7 @@ jobs:
       IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }}
       IDP_TOKEN_URL: ${{ vars.IDP_TOKEN_URL }}
     steps:
-      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
       - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6
         with:
           python-version: 3.11
@@ -39,7 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: CD
     steps:
-      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
         with:
           fetch-depth: 0
       - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.16
+Version: 0.7.18
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/cdf.toml

@@ -23,7 +23,7 @@ data = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.16"
+version = "0.7.18"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/cognite_toolkit/_cdf_tk/apps/_download_app.py

@@ -15,6 +15,7 @@ from cognite_toolkit._cdf_tk.storageio import (
     AssetIO,
     CanvasIO,
     ChartIO,
+    DatapointsIO,
     DataSelector,
     EventIO,
     FileContentIO,
@@ -31,6 +32,7 @@ from cognite_toolkit._cdf_tk.storageio.selectors import (
     CanvasSelector,
     ChartExternalIdSelector,
     ChartSelector,
+    DataPointsDataSetSelector,
     DataSetSelector,
     FileIdentifierSelector,
     InstanceSpaceSelector,
@@ -73,6 +75,16 @@ class HierarchyFormats(str, Enum):
     ndjson = "ndjson"
 
 
+class DatapointFormats(str, Enum):
+    csv = "csv"
+    parquet = "parquet"
+
+
+class DatapointsDataTypes(str, Enum):
+    numeric = "numeric"
+    string = "string"
+
+
 class InstanceFormats(str, Enum):
     ndjson = "ndjson"
 
@@ -108,6 +120,8 @@ class DownloadApp(typer.Typer):
         self.command("events")(self.download_events_cmd)
         self.command("files")(self.download_files_cmd)
         self.command("hierarchy")(self.download_hierarchy_cmd)
+        if Flags.EXTEND_DOWNLOAD.is_enabled():
+            self.command("datapoints")(self.download_datapoints_cmd)
         self.command("instances")(self.download_instances_cmd)
         self.command("charts")(self.download_charts_cmd)
         self.command("canvas")(self.download_canvas_cmd)
@@ -841,6 +855,150 @@ class DownloadApp(typer.Typer):
             )
         )
 
+    @staticmethod
+    def download_datapoints_cmd(
+        dataset: Annotated[
+            str | None,
+            typer.Argument(
+                help="The dataset to download timeseries from. If not provided, an interactive selection will be made.",
+            ),
+        ] = None,
+        start_time: Annotated[
+            str | None,
+            typer.Option(
+                "--start-time",
+                "-s",
+                help="The start time for the datapoints to download. Can be in RFC3339 format or as a relative time (e.g., '1d-ago'). If not provided, all datapoints from the beginning will be downloaded.",
+            ),
+        ] = None,
+        end_time: Annotated[
+            str | None,
+            typer.Option(
+                "--end-time",
+                "-e",
+                help="The end time for the datapoints to download. Can be in RFC3339 format or as a relative time (e.g., '1d-ago'). If not provided, all datapoints up to the latest will be downloaded.",
+            ),
+        ] = None,
+        datapoint_type: Annotated[
+            DatapointsDataTypes,
+            typer.Option(
+                "--data-type",
+                "-d",
+                help="The type of datapoints to download.",
+            ),
+        ] = DatapointsDataTypes.numeric,
+        file_format: Annotated[
+            DatapointFormats,
+            typer.Option(
+                "--format",
+                "-f",
+                help="Format for downloading the datapoints.",
+            ),
+        ] = DatapointFormats.csv,
+        output_dir: Annotated[
+            Path,
+            typer.Option(
+                "--output-dir",
+                "-o",
+                help="Where to download the datapoints.",
+                allow_dash=True,
+            ),
+        ] = DEFAULT_DOWNLOAD_DIR,
+        limit: Annotated[
+            int,
+            typer.Option(
+                "--limit",
+                "-l",
+                help="The maximum number of timeseries to download datapoints from. Use -1 to download all timeseries."
+                "The maximum number of datapoints in total is 10 million and 100 000 per timeseries.",
+                max=10_000_000,
+            ),
+        ] = 1000,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """This command will download Datapoints from CDF into a temporary ."""
+        client = EnvironmentVariables.create_from_environment().get_client()
+        if dataset is None:
+            interactive = TimeSeriesInteractiveSelect(client, "download datapoints")
+            dataset = interactive.select_data_set(allow_empty=False)
+
+            datapoint_type = questionary.select(
+                "Select the type of datapoints to download:",
+                choices=[Choice(title=dt.value, value=dt) for dt in DatapointsDataTypes],
+                default=datapoint_type,
+            ).ask()
+
+            start_time = (
+                questionary.text(
+                    "Enter the start time for the datapoints to download (RFC3339 format or relative time, e.g., '1d-ago'). Leave empty to download from the beginning.",
+                    default=start_time or "",
+                ).ask()
+                or None
+            )
+            end_time = (
+                questionary.text(
+                    "Enter the end time for the datapoints to download (RFC3339 format or relative time, e.g., '1d-ago'). Leave empty to download up to the latest.",
+                    default=end_time or "",
+                ).ask()
+                or None
+            )
+            file_format = questionary.select(
+                "Select format to download the datapoints in:",
+                choices=[Choice(title=format_.value, value=format_) for format_ in DatapointFormats],
+                default=file_format,
+            ).ask()
+            output_dir = Path(
+                questionary.path(
+                    "Where to download the datapoints:", default=str(output_dir), only_directories=True
+                ).ask()
+            )
+            while True:
+                limit_str = questionary.text(
+                    "The maximum number of timeseries to download datapoints from. Use -1 to download all timeseries."
+                    "The maximum number of datapoints in total is 10 million and 100 000 per timeseries.",
+                    default=str(limit),
+                ).ask()
+                if limit_str is None:
+                    raise typer.Abort()
+                try:
+                    limit = int(limit_str)
+                except ValueError:
+                    print("[red]Please enter a valid integer for the limit.[/]")
+                else:
+                    if limit != -1 and limit < 1:
+                        print("[red]Please enter a valid integer greater than 0 or -1 for unlimited.[/]")
+                    else:
+                        break
+                verbose = questionary.confirm(
+                    "Turn on to get more verbose output when running the command?", default=verbose
+                ).ask()
+
+        cmd = DownloadCommand()
+        selector = DataPointsDataSetSelector(
+            data_set_external_id=dataset,
+            start=start_time,
+            end=end_time,
+            data_type=datapoint_type.value,
+        )
+        cmd.run(
+            lambda: cmd.download(
+                selectors=[selector],
+                io=DatapointsIO(client),
+                output_dir=output_dir,
+                file_format=f".{file_format.value}",
+                compression="none",
+                limit=limit if limit != -1 else None,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def download_charts_cmd(
         ctx: typer.Context,

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/cognite_toolkit/_cdf_tk/commands/_download.py

@@ -5,10 +5,21 @@ from pathlib import Path
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.protocols import T_ResourceResponse
-from cognite_toolkit._cdf_tk.storageio import ConfigurableStorageIO, Page, StorageIO, T_Selector, TableStorageIO
+from cognite_toolkit._cdf_tk.storageio import (
+    ConfigurableStorageIO,
+    Page,
+    StorageIO,
+    T_Selector,
+    TableStorageIO,
+)
 from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
 from cognite_toolkit._cdf_tk.utils.file import safe_write, sanitize_filename, yaml_safe_dump
-from cognite_toolkit._cdf_tk.utils.fileio import TABLE_WRITE_CLS_BY_FORMAT, Compression, FileWriter, SchemaColumn
+from cognite_toolkit._cdf_tk.utils.fileio import (
+    TABLE_WRITE_CLS_BY_FORMAT,
+    Compression,
+    FileWriter,
+    SchemaColumn,
+)
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -188,7 +188,11 @@ class MigrationCommand(ToolkitCommand):
                 return None
             responses: Sequence[HTTPMessage]
             if dry_run:
-                responses = [SuccessResponseItems(200, "", [item.source_id for item in data_item])]
+                responses = [
+                    SuccessResponseItems(
+                        status_code=200, body="", content=b"", ids=[item.source_id for item in data_item]
+                    )
+                ]
             else:
                 responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)
 

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/cognite_toolkit/_cdf_tk/feature_flags.py

@@ -55,7 +55,7 @@ class Flags(Enum):
     )
     EXTEND_DOWNLOAD = FlagMetadata(
         visible=True,
-        description="Enables extended download to support downloading file content",
+        description="Enables extended download to support downloading file content and datapoints",
     )
 
     def is_enabled(self) -> bool:

{cognite_toolkit-0.7.16 → cognite_toolkit-0.7.18}/cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -214,7 +214,7 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
             self.client.lookup.files.external_id(list(file_ids))
 
     def _dump_resource(self, canvas: IndustrialCanvas) -> dict[str, JsonVal]:
-        dumped = canvas.as_write().dump()
+        dumped = canvas.as_write().dump(keep_existing_version=False)
         references = dumped.get("containerReferences", [])
         if not isinstance(references, list):
             return dumped
@@ -230,10 +230,18 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
             properties = source["properties"]
             if not isinstance(properties, dict):
                 continue
+            reference_type = properties.get("containerReferenceType")
+            if (
+                reference_type
+                in {
+                    "charts",
+                    "dataGrid",
+                }
+            ):  # These container reference types are special cases with a resourceId statically set to -1, which is why we skip them
+                continue
             resource_id = properties.pop("resourceId", None)
             if not isinstance(resource_id, int):
                 continue
-            reference_type = properties.get("containerReferenceType")
             if reference_type == "asset":
                 external_id = self.client.lookup.assets.external_id(resource_id)
             elif reference_type == "timeseries":