cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/client/testing.py

@@ -21,7 +21,7 @@ from .api.extended_files import ExtendedFileMetadataAPI
 from .api.extended_functions import ExtendedFunctionsAPI
 from .api.extended_raw import ExtendedRawAPI
 from .api.extended_timeseries import ExtendedTimeSeriesAPI
-from .api.infield import InfieldAPI, InfieldConfigAPI
+from .api.infield import InfieldAPI, InFieldCDMConfigAPI, InfieldConfigAPI
 from .api.location_filters import LocationFiltersAPI
 from .api.lookup import (
     AssetLookUpAPI,
@@ -35,7 +35,14 @@ from .api.lookup import (
     SecurityCategoriesLookUpAPI,
     TimeSeriesLookUpAPI,
 )
-from .api.migration import CreatedSourceSystemAPI, InstanceSourceAPI, MigrationAPI, ResourceViewMappingAPI
+from .api.migration import (
+    CreatedSourceSystemAPI,
+    InstanceSourceAPI,
+    LookupAPI,
+    MigrationAPI,
+    MigrationLookupAPI,
+    ResourceViewMappingAPI,
+)
 from .api.project import ProjectAPI
 from .api.robotics import RoboticsAPI
 from .api.robotics.capabilities import CapabilitiesAPI
@@ -45,6 +52,7 @@ from .api.robotics.locations import LocationsAPI as RoboticsLocationsAPI
 from .api.robotics.maps import MapsAPI
 from .api.search import SearchAPI
 from .api.search_config import SearchConfigurationsAPI
+from .api.streams import StreamsAPI
 from .api.token import TokenAPI
 from .api.verify import VerifyAPI
 
@@ -76,6 +84,7 @@ class ToolkitClientMock(CogniteClientMock):
         self.functions.schedules = MagicMock(spec_set=FunctionSchedulesAPI)
         self.infield = MagicMock(spec=InfieldAPI)
         self.infield.config = MagicMock(spec_set=InfieldConfigAPI)
+        self.infield.cdm_config = MagicMock(spec_set=InFieldCDMConfigAPI)
 
         self.project = MagicMock(spec_set=ProjectAPI)
 
@@ -95,6 +104,11 @@ class ToolkitClientMock(CogniteClientMock):
         self.lookup.functions = MagicMock(spec_set=FunctionLookUpAPI)
         self.migration = MagicMock(spec=MigrationAPI)
         self.migration.instance_source = MagicMock(spec_set=InstanceSourceAPI)
+        self.migration.lookup = MagicMock(spec=MigrationLookupAPI)
+        self.migration.lookup.assets = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.events = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.files = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.time_series = MagicMock(spec_set=LookupAPI)
         self.migration.resource_view_mapping = MagicMock(spec_set=ResourceViewMappingAPI)
         self.migration.created_source_system = MagicMock(spec_set=CreatedSourceSystemAPI)
         self.raw = MagicMock(spec=ExtendedRawAPI)
@@ -117,6 +131,8 @@ class ToolkitClientMock(CogniteClientMock):
         self.time_series.data.synthetic = MagicMock(spec_set=SyntheticDatapointsAPI)
         self.time_series.subscriptions = MagicMock(spec_set=DatapointsSubscriptionAPI)
 
+        self.streams = MagicMock(spec=StreamsAPI)
+
         # This is a helper API, not a real API.
         self.token = TokenAPI(self)
         self.verify = MagicMock(spec_set=VerifyAPI)

cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -1,37 +1,36 @@
 from ._download import DownloadCommand
 from ._migrate import (
-    MigrationCanvasCommand,
+    MigrationCommand,
     MigrationPrepareCommand,
 )
 from ._profile import ProfileAssetCentricCommand, ProfileAssetCommand, ProfileRawCommand, ProfileTransformationCommand
 from ._purge import PurgeCommand
 from ._upload import UploadCommand
+from .about import AboutCommand
 from .auth import AuthCommand
 from .build_cmd import BuildCommand
 from .clean import CleanCommand
 from .collect import CollectCommand
 from .deploy import DeployCommand
-from .dump_data import DumpDataCommand
 from .dump_resource import DumpResourceCommand
-from .featureflag import FeatureFlagCommand
 from .init import InitCommand
 from .modules import ModulesCommand
 from .pull import PullCommand
 from .repo import RepoCommand
+from .resources import ResourcesCommand
 from .run import RunFunctionCommand, RunTransformationCommand, RunWorkflowCommand
 
 __all__ = [
+    "AboutCommand",
     "AuthCommand",
     "BuildCommand",
     "CleanCommand",
     "CollectCommand",
     "DeployCommand",
     "DownloadCommand",
-    "DumpDataCommand",
     "DumpResourceCommand",
-    "FeatureFlagCommand",
     "InitCommand",
-    "MigrationCanvasCommand",
+    "MigrationCommand",
     "MigrationPrepareCommand",
     "ModulesCommand",
     "ProfileAssetCentricCommand",
@@ -41,6 +40,7 @@ __all__ = [
     "PullCommand",
     "PurgeCommand",
     "RepoCommand",
+    "ResourcesCommand",
     "RunFunctionCommand",
     "RunTransformationCommand",
     "RunWorkflowCommand",

cognite_toolkit/_cdf_tk/commands/_changes.py

@@ -11,9 +11,8 @@ from packaging.version import Version
 from packaging.version import parse as parse_version
 from rich import print
 
-from cognite_toolkit._cdf_tk.builders import get_loader
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.constants import DOCKER_IMAGE_NAME
-from cognite_toolkit._cdf_tk.data_classes import ModuleDirectories
 from cognite_toolkit._cdf_tk.utils import iterate_modules, read_yaml_file, safe_read, safe_write
 from cognite_toolkit._version import __version__
 
@@ -51,40 +50,6 @@ class ManualChange(Change):
         return ""
 
 
-class SetKindOnFile(AutomaticChange):
-    """Adds the kind to the filename of all resource files.
-
-Before `your_file.yaml`:
-After `your_file.FileMetadata.yaml`:
-    """
-
-    deprecated_from = Version("0.4.0")
-    has_file_changes = True
-
-    def do(self) -> set[Path]:
-        module_directories = ModuleDirectories.load(self._organization_dir)
-        changed: set[Path] = set()
-        for module in module_directories:
-            for resource_folder, source_files in module.source_paths_by_resource_folder.items():
-                for source_file in source_files:
-                    if source_file.suffix not in {".yaml", ".yml"}:
-                        continue
-                    loader, _ = get_loader(source_file, resource_folder, force_pattern=True)
-                    if loader is None:
-                        print(f"Could not find loader for {source_file}")
-                        continue
-                    if source_file.stem.casefold().endswith(loader.kind.casefold()):
-                        continue
-                    new_name = source_file.with_name(f"{source_file.stem}.{loader.kind}{source_file.suffix}")
-                    source_file.rename(new_name)
-                    changed.add(source_file)
-                    for suffix in [".sql", ".csv", ".parquet"]:
-                        if (adjacent_file := source_file.with_suffix(suffix)).exists():
-                            adjacent_file.rename(new_name.with_suffix(suffix))
-                            changed.add(adjacent_file)
-
-        return changed
-
 
 class FixViewBasedLocationFilter(AutomaticChange):
     """The created view-based location filter has been fixed to be compatible with the CDF API.
@@ -362,7 +327,6 @@ After:
 
     def do(self) -> set[Path]:
         # Avoid circular import
-        from .modules import ModulesCommand
 
         system_yaml = self._organization_dir / "_system.yaml"
         if not system_yaml.exists():
@@ -372,11 +336,8 @@ After:
         content = read_yaml_file(system_yaml)
         current_version = content.get("cdf_toolkit_version", __version__)
 
-        cdf_toml_content = ModulesCommand(skip_tracking=True).create_cdf_toml(self._organization_dir)
-        cdf_toml_content = cdf_toml_content.replace(f'version = "{__version__}"', f'version = "{current_version}"')
-
-        cdf_toml_path = Path.cwd() / "cdf.toml"
-        cdf_toml_path.write_text(cdf_toml_content, encoding="utf-8")
+        CDFToml.write(self._organization_dir, version=current_version)
+        cdf_toml_path = Path.cwd() / CDFToml.file_name
         system_yaml.unlink()
         return {cdf_toml_path, system_yaml}
 

cognite_toolkit/_cdf_tk/commands/_download.py

@@ -2,14 +2,24 @@ from collections.abc import Callable, Iterable
 from functools import partial
 from pathlib import Path
 
-from cognite.client.data_classes._base import T_CogniteResource
-
 from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
-from cognite_toolkit._cdf_tk.storageio import ConfigurableStorageIO, Page, StorageIO, T_Selector, TableStorageIO
+from cognite_toolkit._cdf_tk.protocols import T_ResourceResponse
+from cognite_toolkit._cdf_tk.storageio import (
+    ConfigurableStorageIO,
+    Page,
+    StorageIO,
+    T_Selector,
+    TableStorageIO,
+)
 from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
 from cognite_toolkit._cdf_tk.utils.file import safe_write, sanitize_filename, yaml_safe_dump
-from cognite_toolkit._cdf_tk.utils.fileio import TABLE_WRITE_CLS_BY_FORMAT, Compression, FileWriter, SchemaColumn
+from cognite_toolkit._cdf_tk.utils.fileio import (
+    TABLE_WRITE_CLS_BY_FORMAT,
+    Compression,
+    FileWriter,
+    SchemaColumn,
+)
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
@@ -20,7 +30,7 @@ class DownloadCommand(ToolkitCommand):
     def download(
         self,
         selectors: Iterable[T_Selector],
-        io: StorageIO[T_Selector, T_CogniteResource],
+        io: StorageIO[T_Selector, T_ResourceResponse],
         output_dir: Path,
         verbose: bool,
         file_format: str,
@@ -68,7 +78,7 @@ class DownloadCommand(ToolkitCommand):
             with FileWriter.create_from_format(
                 file_format, target_dir, selector.kind, compression_cls, columns=columns
             ) as writer:
-                executor = ProducerWorkerExecutor[Page[T_CogniteResource], list[dict[str, JsonVal]]](
+                executor = ProducerWorkerExecutor[Page[T_ResourceResponse], list[dict[str, JsonVal]]](
                     download_iterable=io.stream_data(selector, limit),
                     process=self.create_data_process(io=io, selector=selector, is_table=is_table),
                     write=partial(writer.write_chunks, filestem=filestem),
@@ -95,7 +105,7 @@ class DownloadCommand(ToolkitCommand):
 
     @staticmethod
     def _get_iteration_count(
-        io: StorageIO[T_Selector, T_CogniteResource],
+        io: StorageIO[T_Selector, T_ResourceResponse],
         selector: T_Selector,
         limit: int | None,
     ) -> int | None:
@@ -126,19 +136,19 @@ class DownloadCommand(ToolkitCommand):
 
     @staticmethod
     def create_data_process(
-        io: StorageIO[T_Selector, T_CogniteResource],
+        io: StorageIO[T_Selector, T_ResourceResponse],
         selector: T_Selector,
         is_table: bool,
-    ) -> Callable[[Page[T_CogniteResource]], list[dict[str, JsonVal]]]:
+    ) -> Callable[[Page[T_ResourceResponse]], list[dict[str, JsonVal]]]:
         """Creates a data processing function based on the IO type and whether the output is a table."""
         if is_table and isinstance(io, TableStorageIO):
 
-            def row_data_process(chunk: Page[T_CogniteResource]) -> list[dict[str, JsonVal]]:
+            def row_data_process(chunk: Page[T_ResourceResponse]) -> list[dict[str, JsonVal]]:
                 return io.data_to_row(chunk.items, selector)
 
             return row_data_process
 
-        def chunk_data_process(data_page: Page[T_CogniteResource]) -> list[dict[str, JsonVal]]:
+        def chunk_data_process(data_page: Page[T_ResourceResponse]) -> list[dict[str, JsonVal]]:
             return io.data_to_json_chunk(data_page.items, selector)
 
         return chunk_data_process

cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py

@@ -1,9 +1,7 @@
-from .canvas import MigrationCanvasCommand
 from .command import MigrationCommand
 from .prepare import MigrationPrepareCommand
 
 __all__ = [
-    "MigrationCanvasCommand",
     "MigrationCommand",
     "MigrationPrepareCommand",
 ]

cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -2,7 +2,6 @@ from collections.abc import Callable, Iterable, Sequence
 from enum import Enum
 from pathlib import Path
 
-from cognite.client.data_classes._base import T_CogniteResource
 from rich import print
 from rich.console import Console
 from rich.table import Table
@@ -20,6 +19,7 @@ from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitMigrationError,
     ToolkitValueError,
 )
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, UploadItem
 from cognite_toolkit._cdf_tk.utils import humanize_collection, safe_write, sanitize_filename
 from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump
@@ -27,7 +27,6 @@ from cognite_toolkit._cdf_tk.utils.fileio import Chunk, CSVWriter, NDJsonWriter,
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import AVAILABLE_STATUS, ProgressTracker, Status
-from cognite_toolkit._cdf_tk.utils.useful_types import T_WriteCogniteResource
 
 from .data_model import INSTANCE_SOURCE_VIEW_ID, MODEL_ID, RESOURCE_VIEW_MAPPING_VIEW_ID
 
@@ -45,8 +44,8 @@ class MigrationCommand(ToolkitCommand):
     def migrate(
         self,
         selected: T_Selector,
-        data: UploadableStorageIO[T_Selector, T_CogniteResource, T_WriteCogniteResource],
-        mapper: DataMapper[T_Selector, T_CogniteResource, T_WriteCogniteResource],
+        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        mapper: DataMapper[T_Selector, T_ResourceResponse, T_ResourceRequest],
         log_dir: Path,
         dry_run: bool = False,
         verbose: bool = False,
@@ -71,9 +70,7 @@ class MigrationCommand(ToolkitCommand):
             NDJsonWriter(log_dir, kind=f"{selected.kind}MigrationIssues", compression=Uncompressed) as log_file,
             HTTPClient(config=data.client.config) as write_client,
         ):
-            executor = ProducerWorkerExecutor[
-                Sequence[T_CogniteResource], Sequence[UploadItem[T_WriteCogniteResource]]
-            ](
+            executor = ProducerWorkerExecutor[Sequence[T_ResourceResponse], Sequence[UploadItem[T_ResourceRequest]]](
                 download_iterable=self._download_iterable(selected, data, tracker),
                 process=self._convert(mapper, data, tracker, log_file),
                 write=self._upload(selected, write_client, data, tracker, log_file, dry_run),
@@ -141,9 +138,9 @@ class MigrationCommand(ToolkitCommand):
     def _download_iterable(
         self,
         selected: T_Selector,
-        data: UploadableStorageIO[T_Selector, T_CogniteResource, T_WriteCogniteResource],
+        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
         tracker: ProgressTracker[str],
-    ) -> Iterable[Sequence[T_CogniteResource]]:
+    ) -> Iterable[Sequence[T_ResourceResponse]]:
         for page in data.stream_data(selected):
             for item in page.items:
                 tracker.set_progress(data.as_id(item), self.Steps.DOWNLOAD, "success")
@@ -151,16 +148,17 @@ class MigrationCommand(ToolkitCommand):
 
     def _convert(
         self,
-        mapper: DataMapper[T_Selector, T_CogniteResource, T_WriteCogniteResource],
-        data: UploadableStorageIO[T_Selector, T_CogniteResource, T_WriteCogniteResource],
+        mapper: DataMapper[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
         tracker: ProgressTracker[str],
         log_file: NDJsonWriter,
-    ) -> Callable[[Sequence[T_CogniteResource]], Sequence[UploadItem[T_WriteCogniteResource]]]:
-        def track_mapping(source: Sequence[T_CogniteResource]) -> list[UploadItem[T_WriteCogniteResource]]:
+    ) -> Callable[[Sequence[T_ResourceResponse]], Sequence[UploadItem[T_ResourceRequest]]]:
+        def track_mapping(source: Sequence[T_ResourceResponse]) -> list[UploadItem[T_ResourceRequest]]:
+            mapped = mapper.map(source)
             issues: list[Chunk] = []
-            targets: list[UploadItem[T_WriteCogniteResource]] = []
-            for item in source:
-                target, issue = mapper.map(item)
+            targets: list[UploadItem[T_ResourceRequest]] = []
+
+            for (target, issue), item in zip(mapped, source):
                 id_ = data.as_id(item)
                 result: Status = "failed" if target is None else "success"
                 tracker.set_progress(id_, step=self.Steps.CONVERT, status=result)
@@ -180,17 +178,21 @@ class MigrationCommand(ToolkitCommand):
         self,
         selected: T_Selector,
         write_client: HTTPClient,
-        target: UploadableStorageIO[T_Selector, T_CogniteResource, T_WriteCogniteResource],
+        target: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
         tracker: ProgressTracker[str],
         log_file: NDJsonWriter,
         dry_run: bool,
-    ) -> Callable[[Sequence[UploadItem[T_WriteCogniteResource]]], None]:
-        def upload_items(data_item: Sequence[UploadItem[T_WriteCogniteResource]]) -> None:
+    ) -> Callable[[Sequence[UploadItem[T_ResourceRequest]]], None]:
+        def upload_items(data_item: Sequence[UploadItem[T_ResourceRequest]]) -> None:
             if not data_item:
                 return None
             responses: Sequence[HTTPMessage]
             if dry_run:
-                responses = [SuccessResponseItems(200, "", [item.source_id for item in data_item])]
+                responses = [
+                    SuccessResponseItems(
+                        status_code=200, body="", content=b"", ids=[item.source_id for item in data_item]
+                    )
+                ]
             else:
                 responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)