cognite-toolkit 0.7.30__py3-none-any.whl → 0.7.51__py3-none-any.whl

cognite_toolkit/_cdf.py

@@ -27,6 +27,7 @@ from cognite_toolkit._cdf_tk.apps import (
     DataApp,
     DevApp,
     DumpApp,
+    ImportApp,
     LandingApp,
     MigrateApp,
     ModulesApp,
@@ -42,7 +43,7 @@ from cognite_toolkit._cdf_tk.constants import HINT_LEAD_TEXT, URL, USE_SENTRY
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitError,
 )
-from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.plugins import Plugins
 from cognite_toolkit._cdf_tk.tracker import Tracker
 from cognite_toolkit._cdf_tk.utils import (
@@ -108,6 +109,9 @@ if Flags.PROFILE.is_enabled():
 if Flags.MIGRATE.is_enabled():
     _app.add_typer(MigrateApp(**default_typer_kws), name="migrate")
 
+if Flags.IMPORT_CMD.is_enabled():
+    _app.add_typer(ImportApp(**default_typer_kws), name="import")
+
 if Plugins.data.value.is_enabled():
     _app.add_typer(DataApp(**default_typer_kws), name="data")
 
@@ -126,11 +130,6 @@ def app() -> NoReturn:
     # --- Main entry point ---
     # Users run 'app()' directly, but that doesn't allow us to control excepton handling:
     try:
-        if FeatureFlag.is_enabled(Flags.IMPORT_CMD):
-            from cognite_toolkit._cdf_tk.prototypes.import_app import import_app
-
-            _app.add_typer(import_app, name="import")
-
         _app()
     except ToolkitError as err:
         if "--verbose" in sys.argv:

cognite_toolkit/_cdf_tk/apps/__init__.py

@@ -4,6 +4,7 @@ from ._data_app import DataApp
 from ._dev_app import DevApp
 from ._download_app import DownloadApp
 from ._dump_app import DumpApp
+from ._import_app import ImportApp
 from ._landing_app import LandingApp
 from ._migrate_app import MigrateApp
 from ._modules_app import ModulesApp
@@ -20,6 +21,7 @@ __all__ = [
     "DevApp",
     "DownloadApp",
     "DumpApp",
+    "ImportApp",
     "LandingApp",
     "MigrateApp",
     "ModulesApp",

cognite_toolkit/_cdf_tk/apps/_core_app.py

@@ -13,8 +13,10 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands import BuildCommand, CleanCommand, DeployCommand
+from cognite_toolkit._cdf_tk.commands.build_v2.build_cmd import BuildCommand as BuildCommandV2
 from cognite_toolkit._cdf_tk.commands.clean import AVAILABLE_DATA_TYPES
 from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.utils import get_cicd_environment
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._version import __version__ as current_version
@@ -207,7 +209,11 @@ class CoreApp(typer.Typer):
         if exit_on_warning:
             print_warning = False
 
-        cmd = BuildCommand(print_warning=print_warning)
+        cmd = (
+            BuildCommandV2(print_warning=print_warning)
+            if Flags.v08.is_enabled()
+            else BuildCommand(print_warning=print_warning)
+        )
         cmd.run(
             lambda: cmd.execute(
                 verbose,

cognite_toolkit/_cdf_tk/apps/_download_app.py

@@ -7,7 +7,7 @@ import typer
 from questionary import Choice
 from rich import print
 
-from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawTable
 from cognite_toolkit._cdf_tk.commands import DownloadCommand
 from cognite_toolkit._cdf_tk.constants import DATA_DEFAULT_DIR
 from cognite_toolkit._cdf_tk.feature_flags import Flags
@@ -616,7 +616,7 @@ class DownloadApp(typer.Typer):
                 file_format = AssetCentricFormats.ndjson
             files = client.files.list(data_set_external_ids=data_sets, limit=limit)
             selector = FileIdentifierSelector(
-                identifiers=tuple([FileInternalID(internal_id=file.id) for file in files])  # type: ignore[call-arg]
+                identifiers=tuple([FileInternalID(internal_id=file.id) for file in files])
             )
             selectors = [selector]
             io = FileContentIO(client, output_dir / sanitize_filename(selector.group))

cognite_toolkit/_cdf_tk/apps/_dump_app.py

@@ -6,7 +6,7 @@ from cognite.client.data_classes import WorkflowVersionId
 from cognite.client.data_classes.data_modeling import DataModelId, ViewId
 from rich import print
 
-from cognite_toolkit._cdf_tk.client.data_classes.search_config import ViewId as SearchConfigViewId
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import ViewId as SearchConfigViewId
 from cognite_toolkit._cdf_tk.commands import DumpResourceCommand
 from cognite_toolkit._cdf_tk.commands.dump_resource import (
     AgentFinder,

cognite_toolkit/_cdf_tk/apps/_import_app.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.commands._import_cmd import ImportTransformationCLI
+from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+
+
+class ImportApp(typer.Typer):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.callback(invoke_without_command=True)(self.main)
+        self.command("transformation-cli")(self.transformation_cli)
+
+    def main(self, ctx: typer.Context) -> None:
+        """PREVIEW FEATURE Import resources into Cognite-Toolkit."""
+        if ctx.invoked_subcommand is None:
+            print("Use [bold yellow]cdf-tk import --help[/] for more information.")
+        return None
+
+    @staticmethod
+    def transformation_cli(
+        source: Path = typer.Argument(..., help="Path to the transformation CLI manifest directory or files."),
+        destination: Path = typer.Argument(..., help="Path to the destination directory."),
+        overwrite: bool = typer.Option(False, help="Overwrite destination if it already exists."),
+        flatten: bool = typer.Option(False, help="Flatten the directory structure."),
+        clean: bool = typer.Option(False, help="Remove the source directory after import."),
+        verbose: bool = typer.Option(False, help="Turn on to get more verbose output when running the command"),
+    ) -> None:
+        """Import transformation CLI manifests into Cognite-Toolkit modules."""
+
+        # We are lazy loading the client as we only need it if we need to look up dataset ids.
+        # This is to ensure the command can be executed without a client if the user does not need to look up dataset ids.
+        # (which is likely 99% of the time)
+        def get_client() -> ToolkitClient:
+            return EnvironmentVariables.create_from_environment().get_client()
+
+        cmd = ImportTransformationCLI(print_warning=True, get_client=get_client)
+        cmd.execute(source, destination, overwrite, flatten, clean, verbose=verbose)

cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -15,10 +15,18 @@ from cognite_toolkit._cdf_tk.commands._migrate.creators import (
     InstanceSpaceCreator,
     SourceSystemCreator,
 )
-from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper, CanvasMapper, ChartMapper
+from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import (
+    AssetCentricMapper,
+    CanvasMapper,
+    ChartMapper,
+    ThreeDAssetMapper,
+    ThreeDMapper,
+)
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
     AnnotationMigrationIO,
     AssetCentricMigrationIO,
+    ThreeDAssetMappingMigrationIO,
+    ThreeDMigrationIO,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     AssetCentricMigrationSelector,
@@ -26,7 +34,11 @@ from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     MigrationCSVFileSelector,
 )
 from cognite_toolkit._cdf_tk.storageio import CanvasIO, ChartIO
-from cognite_toolkit._cdf_tk.storageio.selectors import CanvasExternalIdSelector, ChartExternalIdSelector
+from cognite_toolkit._cdf_tk.storageio.selectors import (
+    CanvasExternalIdSelector,
+    ChartExternalIdSelector,
+    ThreeDModelIdSelector,
+)
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
@@ -36,6 +48,7 @@ from cognite_toolkit._cdf_tk.utils.interactive_select import (
     InteractiveCanvasSelect,
     InteractiveChartSelect,
     ResourceViewMappingInteractiveSelect,
+    ThreeDInteractiveSelect,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
 
@@ -56,6 +69,8 @@ class MigrateApp(typer.Typer):
         self.command("annotations")(self.annotations)
         self.command("canvas")(self.canvas)
         self.command("charts")(self.charts)
+        self.command("3d")(self.three_d)
+        self.command("3d-mappings")(self.three_d_asset_mapping)
         # Uncomment when infield v2 config migration is ready
         # self.command("infield-configs")(self.infield_configs)
 
@@ -853,13 +868,13 @@ class MigrateApp(typer.Typer):
                 "performed to select the Canvas to migrate."
             ),
         ] = None,
-        skip_on_missing_ref: Annotated[
+        allow_missing_ref: Annotated[
             bool,
             typer.Option(
-                "--skip-on-missing-ref",
-                "-s",
-                help="If set, the migration will skip Canvases that reference resources that have not been migrated to data modeling. "
-                "If not set, the migration will continue but the result will exclude the missing references.",
+                "--allow-missing-ref",
+                "-a",
+                help="If set, the command will migrate Canvases that reference resources that have not been migrated to data modeling. "
+                "If not set, the migration will fail if any referenced resource are missing.",
             ),
         ] = False,
         log_dir: Annotated[
@@ -911,7 +926,7 @@ class MigrateApp(typer.Typer):
             lambda: cmd.migrate(
                 selected=selector,
                 data=CanvasIO(client, exclude_existing_version=True),
-                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=skip_on_missing_ref),
+                mapper=CanvasMapper(client, dry_run=dry_run, skip_on_missing_ref=not allow_missing_ref),
                 log_dir=log_dir,
                 dry_run=dry_run,
                 verbose=verbose,
@@ -980,6 +995,166 @@ class MigrateApp(typer.Typer):
             )
         )
 
+    @staticmethod
+    def three_d(
+        ctx: typer.Context,
+        id: Annotated[
+            list[int] | None,
+            typer.Argument(
+                help="The ID of the 3D Model to migrate. If not provided, an interactive selection will be "
+                "performed to select the 3D Models to migrate."
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of "
+                "what would be done is printed. This is useful for checking that all resources referenced by the 3D Models "
+                "have been migrated to the new data modeling resources in CDF.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate 3D Models from Asset-Centric to data modeling in CDF.
+
+        This command expects that the CogniteMigration data model is already deployed, and that the Mapping view
+        is populated with the mapping from Asset-Centric resources to the new data modeling resources.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        selected_ids: list[int]
+        if id:
+            selected_ids = id
+        else:
+            selected_models = ThreeDInteractiveSelect(client, "migrate").select_three_d_models("classic")
+            selected_ids = [model.id for model in selected_models]
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=ThreeDModelIdSelector(ids=tuple(selected_ids)),
+                data=ThreeDMigrationIO(client),
+                mapper=ThreeDMapper(client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
+    @staticmethod
+    def three_d_asset_mapping(
+        ctx: typer.Context,
+        model_id: Annotated[
+            list[int] | None,
+            typer.Argument(
+                help="The IDs of the 3D model to migrate asset mappings for. If not provided, an interactive selection will be "
+                "performed to select the."
+            ),
+        ] = None,
+        object_3D_space: Annotated[
+            str | None,
+            typer.Option(
+                "--object-3d-space",
+                "-o",
+                help="The instance space to ceate the 3D object nodes in.",
+            ),
+        ] = None,
+        cad_node_space: Annotated[
+            str | None,
+            typer.Option(
+                "--cad-node-space",
+                "-c",
+                help="The instance space to create the CAD node nodes in.",
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of what would be done is printed.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate 3D Model Asset Mappings from Asset-Centric to data modeling in CDF.
+
+        This command expects that the selected 3D model has already been migrated to data modeling.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        selected_ids: list[int]
+        if model_id is not None:
+            selected_ids = model_id
+        else:
+            # Interactive selection
+            selected_models = ThreeDInteractiveSelect(client, "migrate").select_three_d_models("dm")
+            selected_ids = [model.id for model in selected_models]
+            space_selector = DataModelingSelect(client, "migrate")
+            object_3D_space = space_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the 3D Object nodes?",
+                include_empty=False,
+            )
+            cad_node_space = space_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the CAD Node nodes?",
+                include_empty=False,
+            )
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [dry_run, verbose]):
+                raise typer.Abort()
+
+        if object_3D_space is None or cad_node_space is None:
+            raise typer.BadParameter(
+                "--object-3d-space and --cad-node-space are required when specifying IDs directly."
+            )
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=ThreeDModelIdSelector(ids=tuple(selected_ids)),
+                data=ThreeDAssetMappingMigrationIO(
+                    client, object_3D_space=object_3D_space, cad_node_space=cad_node_space
+                ),
+                mapper=ThreeDAssetMapper(client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def infield_configs(
         ctx: typer.Context,

cognite_toolkit/_cdf_tk/builders/_function.py

@@ -1,5 +1,8 @@
 import shutil
+import time
 from collections.abc import Callable, Iterable, Sequence
+from pathlib import Path
+from typing import Any
 
 from cognite_toolkit._cdf_tk.builders import Builder
 from cognite_toolkit._cdf_tk.cruds import FunctionCRUD
@@ -10,21 +13,74 @@ from cognite_toolkit._cdf_tk.data_classes import (
     ModuleLocation,
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import (
     FileReadWarning,
+    FunctionRequirementsValidationWarning,
     HighSeverityWarning,
     LowSeverityWarning,
     MediumSeverityWarning,
     ToolkitWarning,
     WarningList,
 )
+from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip
+
+# Maximum number of error lines to display in warnings
+_MAX_ERROR_LINES = 3
 
 
 class FunctionBuilder(Builder):
     _resource_folder = FunctionCRUD.folder_name
 
+    def __init__(self, build_dir: Path, warn: Callable[[ToolkitWarning], None]) -> None:
+        super().__init__(build_dir, warn=warn)
+        # Metrics for telemetry
+        self.validation_count = 0
+        self.validation_failures = 0
+        self.validation_credential_errors = 0
+        self.validation_time_ms = 0
+
+    def _validate_function_requirements(
+        self,
+        requirements_txt: Path,
+        raw_function: dict[str, Any],
+        filepath: Path,
+        external_id: str,
+    ) -> FunctionRequirementsValidationWarning | None:
+        """Validate function requirements.txt using pip dry-run."""
+        start_time = time.time()
+        validation_result = validate_requirements_with_pip(
+            requirements_txt_path=requirements_txt,
+            index_url=raw_function.get("indexUrl"),
+            extra_index_urls=raw_function.get("extraIndexUrls"),
+        )
+        elapsed_ms = int((time.time() - start_time) * 1000)
+        self.validation_count += 1
+        self.validation_time_ms += elapsed_ms
+
+        if validation_result.success:
+            return None
+
+        self.validation_failures += 1
+        if validation_result.is_credential_error:
+            self.validation_credential_errors += 1
+
+        error_detail = validation_result.error_message or "Unknown error"
+        relevant_lines = [line for line in error_detail.strip().split("\n") if line.strip()][-_MAX_ERROR_LINES:]
+        error_detail = "\n      ".join(relevant_lines)
+
+        return FunctionRequirementsValidationWarning(
+            filepath=filepath,
+            function_external_id=external_id,
+            error_details=error_detail,
+            is_credential_error=validation_result.is_credential_error,
+        )
+
     def build(
-        self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None
+        self,
+        source_files: list[BuildSourceFile],
+        module: ModuleLocation,
+        console: Callable[[str], None] | None = None,
     ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]:
         for source_file in source_files:
             if source_file.loaded is None:
@@ -55,7 +111,9 @@ class FunctionBuilder(Builder):
             )
 
     def validate_directory(
-        self, built_resources: BuiltResourceList, module: ModuleLocation
+        self,
+        built_resources: BuiltResourceList,
+        module: ModuleLocation,
     ) -> WarningList[ToolkitWarning]:
         warnings = WarningList[ToolkitWarning]()
         has_config_files = any(resource.kind == FunctionCRUD.kind for resource in built_resources)
@@ -74,7 +132,7 @@ class FunctionBuilder(Builder):
                     f"was not found in {required_location.as_posix()!r}. "
                     f"The file {yaml_source_path.as_posix()!r} is currently "
                     f"considered part of the Function's artifacts and "
-                    f"will not be processed by the Toolkit."
+                    f"will not be processed by the Toolkit.",
                 )
                 warnings.append(warning)
         return warnings
@@ -93,28 +151,42 @@ class FunctionBuilder(Builder):
                 warnings.append(
                     HighSeverityWarning(
                         f"Function in {source_file.source.path.as_posix()!r} has no externalId defined. "
-                        f"This is used to match the function to the function directory."
-                    )
+                        f"This is used to match the function to the function directory.",
+                    ),
                 )
                 continue
             if not function_path:
                 warnings.append(
                     MediumSeverityWarning(
-                        f"Function {external_id} in {source_file.source.path.as_posix()!r} has no function_path defined."
-                    )
+                        f"Function {external_id} in {source_file.source.path.as_posix()!r} has no function_path defined.",
+                    ),
                 )
 
             function_directory = source_file.source.path.with_name(external_id)
 
             if not function_directory.is_dir():
                 raise ToolkitNotADirectoryError(
-                    f"Function directory not found for externalId {external_id} defined in {source_file.source.path.as_posix()!r}."
+                    f"Function directory not found for externalId {external_id} defined in {source_file.source.path.as_posix()!r}.",
+                )
+
+            # Validate requirements.txt if present and feature is enabled
+            if (
+                Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled()
+                and (requirements_txt := function_directory / "requirements.txt").exists()
+            ):
+                warning = self._validate_function_requirements(
+                    requirements_txt,
+                    raw_function,
+                    source_file.source.path,
+                    external_id,
                 )
+                if warning:
+                    warnings.append(warning)
 
             destination = self.build_dir / self.resource_folder / external_id
             if destination.exists():
                 raise ToolkitFileExistsError(
-                    f"Function {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory."
+                    f"Function {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory.",
                 )
             shutil.copytree(function_directory, destination, ignore=shutil.ignore_patterns("__pycache__"))
 

cognite_toolkit/_cdf_tk/builders/_raw.py

@@ -3,7 +3,7 @@ from collections.abc import Callable, Iterable, Sequence
 from typing import Any
 
 from cognite_toolkit._cdf_tk.builders import Builder
-from cognite_toolkit._cdf_tk.client.data_classes.raw import RawDatabase
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawDatabase
 from cognite_toolkit._cdf_tk.cruds import RawDatabaseCRUD, RawTableCRUD, ResourceCRUD
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildDestinationFile,