cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_questionary_style.py

@@ -0,0 +1,16 @@
+import questionary
+
+custom_style_fancy = questionary.Style(
+    [
+        ("qmark", "fg:#673ab7"),  # token in front of the question
+        ("question", "bold"),  # question text
+        ("answer", "fg:#f44336 bold"),  # submitted answer text behind the question
+        ("pointer", "fg:#673ab7 bold"),  # pointer used in select and checkbox prompts
+        ("highlighted", "fg:#673ab7 bold"),  # pointed-at choice in select and checkbox prompts
+        ("selected", "fg:#673ab7"),  # style for a selected item of a checkbox
+        ("separator", "fg:#cc5454"),  # separator in lists
+        ("instruction", ""),  # user instructions for select, rawselect, checkbox
+        ("text", ""),  # plain text
+        ("disabled", "fg:#858585 italic"),  # disabled choices for select and checkbox prompts
+    ]
+)

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -0,0 +1,289 @@
+from collections import Counter
+from collections.abc import Sequence
+from functools import partial
+from pathlib import Path
+
+from cognite.client.data_classes.data_modeling import (
+    ViewId,
+)
+from rich.console import Console
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_SUFFIX, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.cruds import ViewCRUD
+from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
+from cognite_toolkit._cdf_tk.storageio import (
+    T_Selector,
+    UploadableStorageIO,
+    get_upload_io,
+)
+from cognite_toolkit._cdf_tk.storageio._base import TableUploadableStorageIO, UploadItem
+from cognite_toolkit._cdf_tk.storageio.selectors import Selector, load_selector
+from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
+from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
+from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
+from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
+from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
+from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
+
+from ._base import ToolkitCommand
+from .deploy import DeployCommand
+
+
+class UploadCommand(ToolkitCommand):
+    """Command for uploading data to CDF from files in a specified directory.
+
+    This command reads files matching a specific pattern in the input directory,
+    processes them using a StorageIO instance, and uploads the data to CDF.
+
+    Attributes:
+        _MAX_QUEUE_SIZE (int): The maximum size of the queue for processing items.
+            Set to 80 to balance memory usage and processing speed.
+    """
+
+    _MAX_QUEUE_SIZE = 80
+    _UPLOAD = "upload"
+
+    def upload(
+        self,
+        input_dir: Path,
+        client: ToolkitClient,
+        deploy_resources: bool,
+        dry_run: bool,
+        verbose: bool,
+        kind: str | None = None,
+    ) -> None:
+        """Uploads data from files in the specified input directory to CDF.
+
+        Args:
+            input_dir: The directory containing the files to upload. It is expected to
+                have the structure defined below.
+            client: An instance of ToolkitClient to interact with CDF.
+            deploy_resources: If True, deploys resources from the 'resources' subdirectory.
+            dry_run: If True, performs a dry run without actually uploading the data
+                (or deploying resources).
+            verbose: If True, prints detailed information about the upload process.
+            kind: Optional; if provided, only data files of this kind will be processed.
+
+        The expected structure of the input directory is as follows:
+        ```
+        input_dir/
+        ├── resources/                # Optional, only if deploy_resources is True
+        │   ├── raw/
+        │   │   ├── table1.Table.yaml
+        │   │   └── table2.Table.yaml
+        │   └── ...
+        ├── datafile1.kind.ndjson # Data file of a specific kind
+        ├── datafile1.Manifest.yaml       # Manifest for datafile1
+        ├── datafile2.kind2.ndjson # Another data file of the same or different kind
+        ├── datafile2.Manifest.yaml       # Manifest file for datafile2
+        └── ...
+        """
+        console = client.console
+        data_files_by_selector = self._find_data_files(input_dir)
+
+        self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
+
+        data_files_by_selector = self._topological_sort_if_instance_selector(data_files_by_selector, client)
+
+        self._upload_data(data_files_by_selector, client, dry_run, input_dir, console, verbose)
+
+    def _topological_sort_if_instance_selector(
+        self, data_files_by_selector: dict[Selector, list[Path]], client: ToolkitClient
+    ) -> dict[Selector, list[Path]]:
+        """Topologically sorts InstanceSpaceSelectors (if they are present) to determine the order of upload based on container dependencies from the views.
+
+        Args:
+            data_files_by_selector: A dictionary mapping selectors to their data files.
+            client: The cognite client to use for the upload.
+
+        Returns:
+            A dictionary mapping selectors to their data files with necessary preprocessing.
+        """
+        counts = Counter(type(selector) for selector in data_files_by_selector.keys())
+        if counts[InstanceSpaceSelector] <= 1:
+            return data_files_by_selector
+
+        selector_by_view_id: dict[ViewId, Selector] = {}
+        for selector in data_files_by_selector:
+            if isinstance(selector, InstanceSpaceSelector) and selector.view is not None:
+                selector_by_view_id[selector.view.as_id()] = selector
+
+        view_dependencies = ViewCRUD.create_loader(client).topological_sort_container_constraints(
+            list(selector_by_view_id.keys())
+        )
+        prepared_selectors: dict[Selector, list[Path]] = {}
+
+        # Reorder selectors according to the dependency-sorted view list
+        for view_id in view_dependencies:
+            selector = selector_by_view_id[view_id]
+            prepared_selectors[selector] = data_files_by_selector[selector]
+
+        # Preserve selectors that aren't affected by view dependencies
+        # (e.g., raw tables, time series, non-view instance data)
+        for selector in data_files_by_selector.keys():
+            if selector not in prepared_selectors:
+                prepared_selectors[selector] = data_files_by_selector[selector]
+
+        return prepared_selectors
+
+    def _find_data_files(
+        self,
+        input_dir: Path,
+    ) -> dict[Selector, list[Path]]:
+        """Finds data files and their corresponding metadata files in the input directory."""
+        data_files_by_metadata: dict[Selector, list[Path]] = {}
+        for manifest_file in input_dir.glob(f"*{DATA_MANIFEST_SUFFIX}"):
+            selector_or_warning = load_selector(manifest_file)
+            if isinstance(selector_or_warning, ToolkitWarning):
+                self.warn(selector_or_warning)
+                continue
+            selector: Selector = selector_or_warning
+            data_files = selector.find_data_files(input_dir, manifest_file)
+            if not data_files:
+                self.warn(
+                    MediumSeverityWarning(
+                        f"Metadata file {manifest_file.as_posix()!r} has no corresponding data files, skipping.",
+                    )
+                )
+                continue
+            data_files_by_metadata[selector] = data_files
+        return data_files_by_metadata
+
+    def _deploy_resource_folder(
+        self,
+        resource_dir: Path,
+        deploy_resources: bool,
+        client: ToolkitClient,
+        console: Console,
+        dry_run: bool,
+        verbose: bool,
+    ) -> None:
+        """Deploy resources from the specified resource directory if it exists and deployment is enabled."""
+        if deploy_resources and resource_dir.exists():
+            deploy_command = DeployCommand()
+            deploy_results = deploy_command.deploy_all_resources(
+                client=client,
+                build_dir=resource_dir,
+                env_vars=EnvironmentVariables.create_from_environment(),
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+            self.warning_list.extend(deploy_command.warning_list)
+            if deploy_results.has_counts:
+                console.print(deploy_results.counts_table())
+        elif deploy_resources:
+            self.warn(
+                MediumSeverityWarning(
+                    f"Resource directory {resource_dir.as_posix()!r} does not exist, skipping resource deployment."
+                )
+            )
+
+    def _upload_data(
+        self,
+        data_files_by_selector: dict[Selector, list[Path]],
+        client: ToolkitClient,
+        dry_run: bool,
+        input_dir: Path,
+        console: Console,
+        verbose: bool,
+    ) -> None:
+        total_file_count = sum(len(files) for files in data_files_by_selector.values())
+        if verbose:
+            input_dir_display = self._path_as_display_name(input_dir)
+            console.print(f"Found {total_file_count} files to upload in {input_dir_display.as_posix()!r}.")
+        action = "Would upload" if dry_run else "Uploading"
+        with HTTPClient(config=client.config) as upload_client:
+            file_count = 1
+            for selector, datafiles in data_files_by_selector.items():
+                io = self._create_selected_io(selector, datafiles[0], client)
+                if io is None:
+                    continue
+                reader = MultiFileReader(datafiles)
+                if reader.is_table and not isinstance(io, TableUploadableStorageIO):
+                    raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+
+                chunk_count = io.count_chunks(reader)
+                iteration_count = chunk_count // io.CHUNK_SIZE + (1 if chunk_count % io.CHUNK_SIZE > 0 else 0)
+
+                tracker = ProgressTracker[str]([self._UPLOAD])
+                executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
+                    download_iterable=io.read_chunks(reader, selector),
+                    process=partial(io.rows_to_data, selector=selector)
+                    if reader.is_table and isinstance(io, TableUploadableStorageIO)
+                    else io.json_chunk_to_data,
+                    write=partial(
+                        self._upload_items,
+                        upload_client=upload_client,
+                        io=io,
+                        dry_run=dry_run,
+                        selector=selector,
+                        tracker=tracker,
+                        console=console,
+                    ),
+                    iteration_count=iteration_count,
+                    max_queue_size=self._MAX_QUEUE_SIZE,
+                    download_description=f"Reading {selector.display_name!r} files",
+                    process_description="Processing",
+                    write_description=f"{action} {selector.display_name!r}",
+                    console=console,
+                )
+                executor.run()
+                file_count += len(datafiles)
+                executor.raise_on_error()
+                final_action = "Uploaded" if not dry_run else "Would upload"
+                suffix = " successfully" if not dry_run else ""
+                results = tracker.aggregate()
+                success = results.get((self._UPLOAD, "success"), 0)
+                failed = results.get((self._UPLOAD, "failed"), 0)
+                if failed > 0:
+                    suffix += f", {failed:,} failed"
+                console.print(
+                    f"{final_action} {success:,} {selector.display_name} from {len(datafiles)} files{suffix}."
+                )
+
+    @staticmethod
+    def _path_as_display_name(input_path: Path, cwd: Path = Path.cwd()) -> Path:
+        display_name = input_path
+        if input_path.is_relative_to(cwd):
+            display_name = input_path.relative_to(cwd)
+        return display_name
+
+    def _create_selected_io(
+        self, selector: Selector, data_file: Path, client: ToolkitClient
+    ) -> UploadableStorageIO | None:
+        try:
+            io_cls = get_upload_io(selector)
+        except ValueError as e:
+            self.warn(HighSeverityWarning(f"Could not find StorageIO for selector {selector}: {e}"))
+            return None
+        return io_cls(client)
+
+    @classmethod
+    def _upload_items(
+        cls,
+        data_chunk: Sequence[UploadItem[T_ResourceRequest]],
+        upload_client: HTTPClient,
+        io: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        selector: T_Selector,
+        dry_run: bool,
+        tracker: ProgressTracker[str],
+        console: Console,
+    ) -> None:
+        if dry_run:
+            for item in data_chunk:
+                tracker.set_progress(item.source_id, cls._UPLOAD, "success")
+            return
+        results = io.upload_items(data_chunk, upload_client, selector)
+        for message in results:
+            if isinstance(message, SuccessResponseItems):
+                for id_ in message.ids:
+                    tracker.set_progress(id_, step=cls._UPLOAD, status="success")
+            elif isinstance(message, ItemMessage):
+                for id_ in message.ids:
+                    tracker.set_progress(id_, step=cls._UPLOAD, status="failed")
+            else:
+                console.log(f"[red]Unexpected result from upload: {str(message)!r}[/red]")

cognite_toolkit/_cdf_tk/commands/_utils.py

@@ -1,12 +1,6 @@
-from __future__ import annotations
-
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
 from cognite.client.utils.useful_types import SequenceNotStr
 
-from cognite_toolkit._cdf_tk.loaders import (
-    ResourceLoader,
-)
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, T_WritableCogniteResourceList
+from cognite_toolkit._cdf_tk.cruds._base_cruds import T_ID
 
 
 def _print_ids_or_length(resource_ids: SequenceNotStr[T_ID], limit: int = 10) -> str:
@@ -16,22 +10,3 @@ def _print_ids_or_length(resource_ids: SequenceNotStr[T_ID], limit: int = 10) ->
         return f"{resource_ids}"
     else:
         return f"{len(resource_ids)} items"
-
-
-def _remove_duplicates(
-    loaded_resources: T_CogniteResourceList,
-    loader: ResourceLoader[
-        T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-    ],
-) -> tuple[T_CogniteResourceList, list[T_ID]]:
-    seen: set[T_ID] = set()
-    output = loader.list_write_cls([])
-    duplicates: list[T_ID] = []
-    for item in loaded_resources:
-        identifier = loader.get_id(item)
-        if identifier not in seen:
-            output.append(item)
-            seen.add(identifier)
-        else:
-            duplicates.append(identifier)
-    return output, duplicates

cognite_toolkit/_cdf_tk/commands/_virtual_env.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import shutil
 import sys
 import venv

cognite_toolkit/_cdf_tk/commands/about.py

@@ -0,0 +1,221 @@
+import platform
+import sys
+from pathlib import Path
+
+from rich import print
+from rich.table import Table
+
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml, _read_toml
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.constants import clean_name
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.plugins import Plugins
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, MediumSeverityWarning
+from cognite_toolkit._version import __version__
+
+
+class AboutCommand(ToolkitCommand):
+    def execute(self, cwd: Path) -> None:
+        # Version information
+        print(f"\n[bold cyan]Cognite Toolkit[/bold cyan] version: [yellow]{__version__}[/yellow]")
+        print(f"Python version: {sys.version.split()[0]}")
+        print(f"Platform: {platform.system()} {platform.release()}")
+
+        # Check for cdf.toml in the current directory
+        cdf_toml_path = cwd / CDFToml.file_name
+
+        if cdf_toml_path.exists():
+            print(f"\n[bold green]Configuration file found:[/bold green] {cdf_toml_path}")
+
+            cdf_toml = CDFToml.load(cwd)
+
+            # We need to read the raw TOML to get original key names for plugins and alpha flags
+            raw_toml = _read_toml(cdf_toml_path)
+
+            self._check_unrecognized_sections(raw_toml)
+            self._display_plugins(cdf_toml, raw_toml)
+            self._display_alpha_flags(cdf_toml, raw_toml)
+            self._display_additional_config(cdf_toml)
+
+        else:
+            # Search for cdf.toml in subdirectories
+            found_files = self._search_cdf_toml(cwd)
+
+            if found_files:
+                print(f"\n[bold yellow]No cdf.toml found in current directory:[/bold yellow] {cwd}")
+                print("\n[bold]Found cdf.toml files in subdirectories:[/bold]")
+                for file in found_files:
+                    rel_path = file.relative_to(cwd)
+                    print(f"  • {rel_path}")
+                print(f"\n[bold cyan]Hint:[/bold cyan] Move one of these files to {cwd} or navigate to its directory.")
+            else:
+                print("\n[bold yellow]No cdf.toml found[/bold yellow] in current directory or subdirectories.")
+                print(f"Current directory: {cwd}")
+                print("\n[bold cyan]Hint:[/bold cyan] Run [yellow]cdf init[/yellow] to create a new project.")
+
+    def _check_unrecognized_sections(self, raw_toml: dict) -> None:
+        """Check for unrecognized tables in cdf.toml and warn about them."""
+        # Valid top-level tables in cdf.toml
+        valid_tables = {"cdf", "modules", "alpha_flags", "feature_flags", "plugins", "library"}
+
+        # Filter out empty keys, whitespace-only keys, and check for unrecognized tables
+        unrecognized_tables = [key for key in raw_toml.keys() if key and key.strip() and key not in valid_tables]
+
+        if unrecognized_tables:
+            print()
+
+        for table in unrecognized_tables:
+            # Try to find a matching valid table by stripping non-alphabetical characters
+            suggestion = self._find_similar_table(table, valid_tables)
+
+            message = f"Table '{table}' in cdf.toml is not recognized and will have no effect."
+            if suggestion:
+                message += f" Did you mean '{suggestion}'?"
+
+            self.warn(MediumSeverityWarning(message))
+
+    @staticmethod
+    def _find_similar_table(unrecognized: str, valid_tables: set[str]) -> str | None:
+        """Find a similar valid table by comparing alphabetical characters only.
+
+        Returns None if the unrecognized table is already valid or if no similar match is found.
+        """
+        # If it's already a valid table, return None (no suggestion needed)
+        if unrecognized in valid_tables:
+            return None
+
+        # Keep only alphabetical characters and lowercase
+        normalized_unrecognized = "".join(c for c in unrecognized if c.isalpha()).lower()
+
+        # First, try exact match (after normalization)
+        for valid in valid_tables:
+            normalized_valid = "".join(c for c in valid if c.isalpha()).lower()
+            if normalized_unrecognized == normalized_valid:
+                return valid
+
+        # If no match, check for singular/plural variations (missing 's')
+        for valid in valid_tables:
+            normalized_valid = "".join(c for c in valid if c.isalpha()).lower()
+
+            # Check if adding 's' to unrecognized matches valid (e.g., "plugin" -> "plugins")
+            if normalized_unrecognized + "s" == normalized_valid:
+                return valid
+
+        return None
+
+    def _display_plugins(self, cdf_toml: CDFToml, raw_toml: dict) -> None:
+        """Display all available plugins and their status."""
+        table = Table(title="Plugins", show_header=True)
+        table.add_column("Plugin", justify="left", style="cyan")
+        table.add_column("Status", justify="center")
+        table.add_column("Description", justify="left")
+
+        # Track which plugins we've seen
+        seen_plugins = set()
+
+        # Show all plugins from the enum
+        for plugin in Plugins:
+            plugin_name = plugin.value.name
+            cleaned_key = clean_name(plugin_name)
+            seen_plugins.add(cleaned_key)
+
+            is_enabled = cdf_toml.plugins.get(cleaned_key, False)
+            if is_enabled:
+                status = "[green]✓ enabled[/green]"
+            else:
+                status = "[dim]○ disabled[/dim]"
+
+            table.add_row(plugin_name, status, plugin.value.description)
+
+        print()
+        print(table)
+
+        # Show any unrecognized plugins from cdf.toml using original key names
+        raw_plugins = raw_toml.get("plugins", {})
+        unrecognized = []
+        for original_key, value in raw_plugins.items():
+            cleaned_key = clean_name(original_key)
+            if cleaned_key not in seen_plugins:
+                unrecognized.append((original_key, value))
+
+        for original_key, is_enabled in unrecognized:
+            status = "enabled" if is_enabled else "disabled"
+            self.warn(
+                LowSeverityWarning(f"Plugin '{original_key}' in cdf.toml is not recognized and will have no effect.")
+            )
+
+    def _display_alpha_flags(self, cdf_toml: CDFToml, raw_toml: dict) -> None:
+        """Display available alpha flags and their status."""
+        table = Table(title="Alpha Flags", show_header=True)
+        table.add_column("Flag", justify="left", style="yellow")
+        table.add_column("Status", justify="center")
+        table.add_column("Description", justify="left")
+
+        # Track which flags we've seen
+        seen_flags = set()
+
+        # Show flags from the enum that are either enabled or visible
+        for flag in Flags:
+            cleaned_key = clean_name(flag.name)
+            seen_flags.add(cleaned_key)
+
+            is_enabled = cdf_toml.alpha_flags.get(cleaned_key, False)
+
+            # Only show if enabled or visible
+            if is_enabled or flag.value.visible:
+                # Convert enum name to kebab-case for display
+                display_name = flag.name.lower().replace("_", "-")
+
+                if is_enabled:
+                    status = "[green]✓ enabled[/green]"
+                else:
+                    status = "[dim]○ disabled[/dim]"
+
+                table.add_row(display_name, status, flag.value.description)
+
+        print()
+        print(table)
+
+        # Show any unrecognized flags from cdf.toml using original key names
+        raw_flags = raw_toml.get("alpha_flags", {})
+        unrecognized = []
+        for original_key, value in raw_flags.items():
+            cleaned_key = clean_name(original_key)
+            if cleaned_key not in seen_flags:
+                unrecognized.append((original_key, value))
+
+        for original_key, is_enabled in unrecognized:
+            status = "enabled" if is_enabled else "disabled"
+            self.warn(
+                LowSeverityWarning(
+                    f"Alpha flag '{original_key}' in cdf.toml is not recognized and will have no effect."
+                )
+            )
+
+    def _display_additional_config(self, cdf_toml: CDFToml) -> None:
+        """Display additional configuration information."""
+        print("\n[bold]Additional Configuration:[/bold]")
+
+        print(f"  Default environment: [cyan]{cdf_toml.cdf.default_env}[/cyan]")
+
+        if cdf_toml.cdf.has_user_set_default_org:
+            print(f"  Default organization dir: [cyan]{cdf_toml.cdf.default_organization_dir}[/cyan]")
+
+        if cdf_toml.cdf.file_encoding:
+            print(f"  File encoding: [cyan]{cdf_toml.cdf.file_encoding}[/cyan]")
+
+        print(f"  Modules version: [cyan]{cdf_toml.modules.version}[/cyan]")
+
+        if cdf_toml.libraries:
+            print(f"  Configured libraries: [cyan]{len(cdf_toml.libraries)}[/cyan]")
+            for lib_name, lib_config in cdf_toml.libraries.items():
+                print(f"    • {lib_name}: [dim]{lib_config.url}[/dim]")
+
+    def _search_cdf_toml(self, cwd: Path) -> list[Path]:
+        """Search for cdf.toml files in immediate subdirectories (one level down)."""
+        try:
+            return sorted(
+                [potential_file for potential_file in cwd.glob(f"*/{CDFToml.file_name}") if potential_file.is_file()]
+            )
+        except PermissionError:
+            return []