cognite-toolkit 0.6.78__tar.gz → 0.6.79__tar.gz

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.78
+Version: 0.6.79
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/cdf.toml

@@ -30,4 +30,4 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.78"
+version = "0.6.79"

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/cognite_toolkit/_builtin_modules/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.78"
+version = "0.6.79"
 
 
 [plugins]

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -8,13 +8,13 @@ from cognite.client.data_classes.data_modeling import ContainerId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands import (
-    MigrateFilesCommand,
     MigrationCanvasCommand,
     MigrationPrepareCommand,
 )
 from cognite_toolkit._cdf_tk.commands._migrate import MigrationCommand
 from cognite_toolkit._cdf_tk.commands._migrate.adapter import (
     AssetCentricMigrationIOAdapter,
+    FileMetaIOAdapter,
     MigrateDataSetSelector,
     MigrationCSVFileSelector,
     MigrationSelector,
@@ -581,11 +581,12 @@ class MigrateApp(typer.Typer):
             )
         )
 
-    @staticmethod
+    @classmethod
     def files(
+        cls,
         ctx: typer.Context,
         mapping_file: Annotated[
-            Path,
+            Path | None,
             typer.Option(
                 "--mapping-file",
                 "-m",
@@ -593,7 +594,51 @@ class MigrateApp(typer.Typer):
                 "This file is expected to have the following columns: [id, dataSetId, space, externalId]."
                 "The dataSetId is optional, and can be skipped. If it is set, it is used to check the access to the dataset.",
             ),
-        ],
+        ] = None,
+        data_set_id: Annotated[
+            str | None,
+            typer.Option(
+                "--data-set-id",
+                "-s",
+                help="The data set ID to select for the files to migrate. If not provided, the dataSetId from the mapping file is used. "
+                "If neither is provided, the default data set for the project is used.",
+            ),
+        ] = None,
+        ingestion_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--ingestion-mapping",
+                "-i",
+                help="The ingestion mapping to use for the migrated files. If not provided, "
+                "the default mapping to CogniteFile in CogniteCore will be used.",
+            ),
+        ] = None,
+        consumption_view: Annotated[
+            str | None,
+            typer.Option(
+                "--consumption-view",
+                "-c",
+                help="The consumption view to assign to the migrated files Given as space:externalId/version. "
+                "This will be used in Canvas to select which view to use when migrating timeseries. If not provided, "
+                "CogniteFile in CogniteCore will be used.",
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where logs will be stored. If the directory does not exist, it will be created.",
+            ),
+        ] = Path(f"migration_logs_{TODAY!s}"),
+        skip_linking: Annotated[
+            bool,
+            typer.Option(
+                "--skip-linking",
+                "-x",
+                help="If set, the migration will not create links between the old FileMetadata and the new CogniteFile.",
+            ),
+        ] = False,
         dry_run: Annotated[
             bool,
             typer.Option(
@@ -612,13 +657,33 @@ class MigrateApp(typer.Typer):
         ] = False,
     ) -> None:
         """Migrate Files to CogniteFiles."""
-
         client = EnvironmentVariables.create_from_environment().get_client(enable_set_pending_ids=True)
-        cmd = MigrateFilesCommand()
+
+        selected, dry_run, verbose = cls._prepare_asset_centric_arguments(
+            client=client,
+            mapping_file=mapping_file,
+            data_set_id=data_set_id,
+            consumption_view=consumption_view,
+            ingestion_mapping=ingestion_mapping,
+            dry_run=dry_run,
+            verbose=verbose,
+            kind="FileMetadata",
+            resource_type="file",
+            container_id=ContainerId("cdf_cdm", "CogniteFile"),
+        )
+        cmd = MigrationCommand()
+
+        if data_set_id is None:
+            skip_linking = not questionary.confirm(
+                "Do you want to link old and new Files?", default=not skip_linking
+            ).ask()
+
         cmd.run(
-            lambda: cmd.migrate_files(
-                client,
-                mapping_file=mapping_file,
+            lambda: cmd.migrate(
+                selected=selected,
+                data=FileMetaIOAdapter(client, skip_linking=skip_linking),
+                mapper=AssetCentricMapper(client),
+                log_dir=log_dir,
                 dry_run=dry_run,
                 verbose=verbose,
             )

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/cognite_toolkit/_cdf_tk/commands/_migrate/adapter.py

@@ -326,7 +326,7 @@ class LinkingAdapter(
         return results
 
 
-class FileMetaAdapter(
+class FileMetaIOAdapter(
     LinkingAdapter[
         str,
         FileMetadataWrite,

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py

@@ -32,6 +32,7 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
         "labels": "tags",
         "mimeType": "mimeType",
         "directory": "directory",
+        "assetIds": "assets",
         "sourceCreatedTime": "sourceCreatedTime",
         "sourceModifiedTime": "sourceUpdatedTime",
     }

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.78
+      image: cognite/toolkit:0.6.79
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.78
+      image: cognite/toolkit:0.6.79
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit-0.6.79/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.79"

cognite_toolkit-0.6.79/last_changelog_entry.md

@@ -0,0 +1,10 @@
+## cdf 
+
+### Changed
+
+- [alpha] The command `cdf migrate files` now support migrating a
+dataset. You can also now chose to skip the linking to existing files.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.79/last_git_message.txt

@@ -0,0 +1,32 @@
+[CDF-25971] 🗂 Migrate filemetadata (#2135)
+
+# Description
+
+**Context**: This is part of the `alpha` migration tooling in Toolkit,
+for overview [see
+here](https://cognitedata.atlassian.net/wiki/spaces/~170890702/pages/5455773697/Migration+Tooling+in+Toolkit).
+This is step 3 of that process.
+
+This is the fourth of the 'v2' migration commands. This uses the new
+generic interface to migrate files and linking them.
+
+<img width="1348" height="385" alt="image"
+src="https://github.com/user-attachments/assets/d8837946-0fb9-4ec6-8fc9-5760550cddab"
+/>
+
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Changed
+
+- [alpha] The command `cdf migrate files` now support migrating a
+dataset. You can also now chose to skip the linking to existing files.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.79/last_version.txt

@@ -0,0 +1 @@
+0.6.78

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.78"
+version = "0.6.79"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/tests/test_integration/test_commands/test_migrate_assets.py

@@ -10,13 +10,19 @@ from cognite.client.exceptions import CogniteAPIError
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands._migrate.adapter import (
     AssetCentricMigrationIOAdapter,
+    FileMetaIOAdapter,
     MigrateDataSetSelector,
     MigrationCSVFileSelector,
     TimeSeriesIOAdapter,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.command import MigrationCommand
 from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper
-from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import ASSET_ID, EVENT_ID, TIME_SERIES_ID
+from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
+    ASSET_ID,
+    EVENT_ID,
+    FILE_METADATA_ID,
+    TIME_SERIES_ID,
+)
 from cognite_toolkit._cdf_tk.storageio import AssetIO, EventIO
 from tests.test_integration.conftest import HierarchyMinimal
 from tests.test_integration.constants import RUN_UNIQUE_ID
@@ -154,3 +160,27 @@ class TestMigrateTimeSeriesCommand:
         results = progress.aggregate()
         expected_results = {(step, "success"): 1 for step in cmd.Steps.list()}
         assert results == expected_results
+
+
+class TestMigrateFileMetadataCommand:
+    def test_migrate_file_metadata_by_dataset_dry_run(
+        self, toolkit_client: ToolkitClient, migration_hierarchy_minimal: HierarchyMinimal, tmp_path: Path
+    ) -> None:
+        client = toolkit_client
+        hierarchy = migration_hierarchy_minimal
+        cmd = MigrationCommand(skip_tracking=True, silent=True)
+        progress = cmd.migrate(
+            selected=MigrateDataSetSelector(
+                kind="FileMetadata",
+                data_set_external_id=hierarchy.dataset.external_id,
+                ingestion_mapping=FILE_METADATA_ID,
+                preferred_consumer_view=ViewId("cdf_cdm", "CogniteFile", "v1"),
+            ),
+            data=FileMetaIOAdapter(client, skip_linking=False),
+            mapper=AssetCentricMapper(client),
+            log_dir=tmp_path,
+            dry_run=True,
+        )
+        results = progress.aggregate()
+        expected_results = {(step, "success"): 1 for step in cmd.Steps.list()}
+        assert results == expected_results

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/tests/test_integration/test_commands/test_migrate_files.py

@@ -12,7 +12,7 @@ from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.extended_timeseries import ExtendedTimeSeriesList
 from cognite_toolkit._cdf_tk.commands import MigrateFilesCommand
 from cognite_toolkit._cdf_tk.commands._migrate.adapter import (
-    FileMetaAdapter,
+    FileMetaIOAdapter,
     MigrationCSVFileSelector,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.command import MigrationCommand
@@ -124,7 +124,7 @@ class TestMigrateFilesCommand:
         cmd = MigrationCommand(skip_tracking=True, silent=True)
         results = cmd.migrate(
             selected=MigrationCSVFileSelector(datafile=input_file, kind="file"),
-            data=FileMetaAdapter(client, skip_linking=False),
+            data=FileMetaIOAdapter(client, skip_linking=False),
             mapper=AssetCentricMapper(client),
             log_dir=tmp_path / "logs",
             dry_run=False,

{cognite_toolkit-0.6.78 → cognite_toolkit-0.6.79}/uv.lock

@@ -235,7 +235,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.6.78"
+version = "0.6.79"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.6.78/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.6.78"

cognite_toolkit-0.6.78/last_changelog_entry.md

@@ -1,11 +0,0 @@
-## cdf 
-
-### Changed
-
-- [alpha] The command `cdf migrate timeseries` now support migrating a
-dataset. You can also now chose to skip the linking to existing
-timeseries.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.78/last_git_message.txt

@@ -1,39 +0,0 @@
-[CDF-25970] 😄Migrate timeseries (#2134)
-
-# Description
-
-
-Stacked on #2127, #2126, #2095, #2128
-
-**Context**: This is part of the `alpha` migration tooling in Toolkit,
-for overview [see
-here](https://cognitedata.atlassian.net/wiki/spaces/~170890702/pages/5455773697/Migration+Tooling+in+Toolkit).
-This is step 3 of that process.
-
-This is the third of the 'v2' migration commands. This uses the new
-generic interface to migrate timeseries and linking them..
-<img width="1778" height="369" alt="image"
-src="https://github.com/user-attachments/assets/9c057989-8750-4979-b877-744e9a136b88"
-/>
-
-<img width="1455" height="262" alt="image"
-src="https://github.com/user-attachments/assets/37323f10-c544-4037-a09c-a86c461f1045"
-/>
-
-
-## Changelog
-
-- [x] Patch
-- [ ] Skip
-
-## cdf
-
-### Changed
-
-- [alpha] The command `cdf migrate timeseries` now support migrating a
-dataset. You can also now chose to skip the linking to existing
-timeseries.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.78/last_version.txt

@@ -1 +0,0 @@
-0.6.77