cognite-toolkit 0.7.39__py3-none-any.whl → 0.7.51__py3-none-any.whl

cognite_toolkit/_cdf_tk/storageio/_file_content.py

@@ -6,26 +6,28 @@ from pathlib import Path
 from typing import cast
 
 import httpx
-from cognite.client.data_classes import FileMetadata, FileMetadataWrite
-from cognite.client.data_classes.data_modeling import NodeId, ViewId
+from cognite.client.data_classes.data_modeling import ViewId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.cruds import FileMetadataCRUD
-from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
-from cognite_toolkit._cdf_tk.protocols import ResourceResponseProtocol
-from cognite_toolkit._cdf_tk.utils import sanitize_filename
-from cognite_toolkit._cdf_tk.utils.collection import chunker, chunker_sequence
-from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
-from cognite_toolkit._cdf_tk.utils.http_client import (
-    DataBodyRequest,
+from cognite_toolkit._cdf_tk.client.http_client import (
     ErrorDetails,
-    FailedResponse,
+    FailedResponse2,
     FailedResponseItems,
     HTTPClient,
     HTTPMessage,
-    ResponseList,
+    HTTPResult2,
+    RequestMessage2,
     SimpleBodyRequest,
+    SuccessResponse2,
 )
+from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeReference
+from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataRequest, FileMetadataResponse
+from cognite_toolkit._cdf_tk.cruds import FileMetadataCRUD
+from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
+from cognite_toolkit._cdf_tk.protocols import ResourceResponseProtocol
+from cognite_toolkit._cdf_tk.utils import sanitize_filename
+from cognite_toolkit._cdf_tk.utils.collection import chunker, chunker_sequence
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
 from ._base import Page, UploadableStorageIO, UploadItem
@@ -45,21 +47,24 @@ COGNITE_FILE_VIEW = ViewId("cdf_cdm", "CogniteFile", "v1")
 
 
 @dataclass
-class UploadFileContentItem(UploadItem[FileMetadataWrite]):
+class UploadFileContentItem(UploadItem[FileMetadataRequest]):
     file_path: Path
     mime_type: str
 
+    def dump(self) -> JsonVal:
+        return self.item.dump(camel_case=True, exclude_extra=True)
+
 
 @dataclass
 class MetadataWithFilePath(ResourceResponseProtocol):
-    metadata: FileMetadata
+    metadata: FileMetadataResponse
     file_path: Path
 
-    def as_write(self) -> FileMetadataWrite:
-        return self.metadata.as_write()
+    def as_write(self) -> FileMetadataRequest:
+        return self.metadata.as_request_resource()
 
 
-class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePath, FileMetadataWrite]):
+class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePath, FileMetadataRequest]):
     SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
     SUPPORTED_COMPRESSIONS = frozenset({".gz"})
     CHUNK_SIZE = 10
@@ -116,7 +121,7 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
                 )
             yield Page(items=downloaded_files, worker_id="Main")
 
-    def _retrieve_metadata(self, identifiers: Sequence[FileIdentifier]) -> Sequence[FileMetadata] | None:
+    def _retrieve_metadata(self, identifiers: Sequence[FileIdentifier]) -> Sequence[FileMetadataResponse] | None:
         config = self.client.config
         responses = self.client.http_client.request_with_retries(
             message=SimpleBodyRequest(
@@ -137,12 +142,11 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
         items_data = body.get("items", [])
         if not isinstance(items_data, list):
             return None
-        # MyPy does not understand that JsonVal is valid dict[Any, Any]
-        return [FileMetadata._load(item) for item in items_data]  # type: ignore[arg-type]
+        return [FileMetadataResponse.model_validate(item) for item in items_data]
 
     @staticmethod
-    def _as_metadata_map(metadata: Sequence[FileMetadata]) -> dict[FileIdentifier, FileMetadata]:
-        identifiers_map: dict[FileIdentifier, FileMetadata] = {}
+    def _as_metadata_map(metadata: Sequence[FileMetadataResponse]) -> dict[FileIdentifier, FileMetadataResponse]:
+        identifiers_map: dict[FileIdentifier, FileMetadataResponse] = {}
         for item in metadata:
             if item.id is not None:
                 identifiers_map[FileInternalID(internal_id=item.id)] = item
@@ -158,9 +162,9 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
                 ] = item
         return identifiers_map
 
-    def _create_filepath(self, meta: FileMetadata, selector: FileIdentifierSelector) -> Path:
+    def _create_filepath(self, meta: FileMetadataResponse, selector: FileIdentifierSelector) -> Path:
         # We now that metadata always have name set
-        filename = Path(sanitize_filename(cast(str, meta.name)))
+        filename = Path(sanitize_filename(meta.name))
         if len(filename.suffix) == 0 and meta.mime_type:
             if mime_ext := mimetypes.guess_extension(meta.mime_type):
                 filename = filename.with_suffix(mime_ext)
@@ -245,12 +249,12 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
             )
         return result
 
-    def json_to_resource(self, item_json: dict[str, JsonVal]) -> FileMetadataWrite:
+    def json_to_resource(self, item_json: dict[str, JsonVal]) -> FileMetadataRequest:
         return self._crud.load_resource(item_json)
 
     def upload_items(
         self,
-        data_chunk: Sequence[UploadItem[FileMetadataWrite]],
+        data_chunk: Sequence[UploadItem[FileMetadataRequest]],
         http_client: HTTPClient,
         selector: FileContentSelector | None = None,
     ) -> Sequence[HTTPMessage]:
@@ -271,8 +275,8 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
                 continue
 
             content_bytes = item.file_path.read_bytes()
-            upload_response = http_client.request_with_retries(
-                message=DataBodyRequest(
+            upload_response = http_client.request_single_retries(
+                RequestMessage2(
                     endpoint_url=upload_url,
                     method="PUT",
                     content_type=item.mime_type,
@@ -280,21 +284,21 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
                     content_length=len(content_bytes),
                 )
             )
-            results.extend(upload_response.as_item_responses(item.as_id()))
+            results.append(upload_response.as_item_response(item.as_id()))
         return results
 
     def _upload_url_asset_centric(
         self, item: UploadFileContentItem, http_client: HTTPClient, results: MutableSequence[HTTPMessage]
     ) -> str | None:
-        responses = http_client.request_with_retries(
-            message=SimpleBodyRequest(
+        response = http_client.request_single_retries(
+            message=RequestMessage2(
                 endpoint_url=http_client.config.create_api_url(self.UPLOAD_ENDPOINT),
                 method="POST",
                 # MyPy does not understand that .dump is valid json
                 body_content=item.dump(),  # type: ignore[arg-type]
             )
         )
-        return self._parse_upload_link_response(responses, item, results)
+        return self._parse_upload_link_response(response, item, results)
 
     def _upload_url_data_modeling(
         self,
@@ -320,30 +324,28 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
 
         """
         # We know that instance_id is always set for data modeling uploads
-        instance_id = cast(NodeId, item.item.instance_id)
-        responses = http_client.request_with_retries(
-            message=SimpleBodyRequest(
+        instance_id = cast(NodeReference, item.item.instance_id)
+        response = http_client.request_single_retries(
+            message=RequestMessage2(
                 endpoint_url=http_client.config.create_api_url("/files/uploadlink"),
                 method="POST",
-                body_content={"items": [{"instanceId": instance_id.dump(include_instance_type=False)}]},  # type: ignore[dict-item]
+                body_content={"items": [{"instanceId": instance_id.dump()}]},
             )
         )
-        # We know there is only one response since we only requested one upload link
-        response = responses[0]
-        if isinstance(response, FailedResponse) and response.error.missing and not created_node:
+        if isinstance(response, FailedResponse2) and response.error.missing and not created_node:
             if self._create_cognite_file_node(instance_id, http_client, item.as_id(), results):
                 return self._upload_url_data_modeling(item, http_client, results, created_node=True)
             else:
                 return None
 
-        return self._parse_upload_link_response(responses, item, results)
+        return self._parse_upload_link_response(response, item, results)
 
     @classmethod
     def _create_cognite_file_node(
-        cls, instance_id: NodeId, http_client: HTTPClient, upload_id: str, results: MutableSequence[HTTPMessage]
+        cls, instance_id: NodeReference, http_client: HTTPClient, upload_id: str, results: MutableSequence[HTTPMessage]
     ) -> bool:
-        node_creation = http_client.request_with_retries(
-            message=SimpleBodyRequest(
+        node_creation = http_client.request_single_retries(
+            message=RequestMessage2(
                 endpoint_url=http_client.config.create_api_url("/models/instances"),
                 method="POST",
                 body_content={
@@ -363,25 +365,33 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, MetadataWithFilePat
                 },
             )
         )
-        try:
-            _ = node_creation.get_first_body()
-        except ValueError:
-            results.extend(node_creation.as_item_responses(upload_id))
-            return False
-        return True
+        if isinstance(node_creation, SuccessResponse2):
+            # Node created successfully
+            return True
+        results.append(node_creation.as_item_response(instance_id))
+        return False
 
     @classmethod
     def _parse_upload_link_response(
-        cls, responses: ResponseList, item: UploadFileContentItem, results: MutableSequence[HTTPMessage]
+        cls, response: HTTPResult2, item: UploadFileContentItem, results: MutableSequence[HTTPMessage]
     ) -> str | None:
+        if not isinstance(response, SuccessResponse2):
+            results.append(response.as_item_response(item.as_id()))
+            return None
         try:
-            body = responses.get_first_body()
+            body = response.body_json
         except ValueError:
-            results.extend(responses.as_item_responses(item.as_id()))
+            results.append(
+                FailedResponseItems(
+                    status_code=response.status_code,
+                    body=response.body,
+                    error=ErrorDetails(code=response.status_code, message="Invalid JSON response"),
+                    ids=[item.as_id()],
+                )
+            )
             return None
-
         if "items" in body and isinstance(body["items"], list) and len(body["items"]) > 0:
-            body = body["items"][0]  # type: ignore[assignment]
+            body = body["items"][0]
         try:
             upload_url = cast(str, body["uploadUrl"])
         except (KeyError, IndexError):

cognite_toolkit/_cdf_tk/storageio/_instances.py

@@ -17,7 +17,7 @@ from cognite.client.utils._identifier import InstanceId
 
 from cognite_toolkit._cdf_tk import constants
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.instances import InstanceList
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.instances import InstanceList
 from cognite_toolkit._cdf_tk.cruds import ContainerCRUD, SpaceCRUD, ViewCRUD
 from cognite_toolkit._cdf_tk.utils import sanitize_filename
 from cognite_toolkit._cdf_tk.utils.cdf import iterate_instances

cognite_toolkit/_cdf_tk/storageio/_raw.py

@@ -4,12 +4,12 @@ from uuid import uuid4
 
 from cognite.client.data_classes import Row, RowWrite
 
+from cognite_toolkit._cdf_tk.client.http_client import HTTPClient, HTTPMessage, ItemsRequest
 from cognite_toolkit._cdf_tk.cruds import RawDatabaseCRUD, RawTableCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.utils import sanitize_filename
 from cognite_toolkit._cdf_tk.utils.collection import chunker
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
-from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemsRequest
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
 from ._base import (

cognite_toolkit/_cdf_tk/storageio/logger.py

@@ -0,0 +1,162 @@
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from collections.abc import Sequence
+from threading import Lock
+from typing import Literal, TypeAlias
+
+from pydantic import BaseModel
+from pydantic.alias_generators import to_camel
+
+from cognite_toolkit._cdf_tk.utils.fileio import NDJsonWriter
+
+
+class LogEntry(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
+    """Represents a log entry for tracking storage I/O operations."""
+
+    id: str
+
+
+OperationStatus: TypeAlias = Literal["success", "failure", "unchanged", "pending"]
+
+
+class ItemTracker:
+    """Tracks issues accumulated for a single item during pipeline processing."""
+
+    def __init__(self, item_id: str) -> None:
+        self.item_id = item_id
+        self.issues: list[str] = []
+
+    def add_issue(self, issue: str) -> None:
+        """Add an issue encountered during processing."""
+        self.issues.append(issue)
+
+
+class OperationTracker(ABC):
+    """Abstract base class for operation trackers."""
+
+    @abstractmethod
+    def add_issue(self, item_id: str, issue: str) -> None:
+        """Add an issue to an item."""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def finalize_item(self, item_id: str | list[str], status: OperationStatus) -> None:
+        """Finalize an item with its final status."""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_status_counts(self) -> dict[OperationStatus, int]:
+        """Get counts per final status."""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_issue_counts(self, status: OperationStatus) -> dict[str, int]:
+        """Get issue counts, optionally filtered by status."""
+        raise NotImplementedError()
+
+
+class NoOpTracker(OperationTracker):
+    """A no-op tracker that does nothing."""
+
+    def add_issue(self, item_id: str, issue: str) -> None:
+        """No-op: Discard the issue."""
+        pass
+
+    def finalize_item(self, item_id: str | list[str], status: OperationStatus) -> None:
+        """No-op: Do nothing."""
+        pass
+
+    def get_status_counts(self) -> dict[OperationStatus, int]:
+        """Return empty status counts."""
+        return {}
+
+    def get_issue_counts(self, status: OperationStatus) -> dict[str, int]:
+        """Return empty issue counts."""
+        return {}
+
+
+class MemoryOperationTracker(OperationTracker):
+    """Tracks the overall operation progress and issues across multiple items.
+
+    Tracks counts of final statuses and issues per status.
+    """
+
+    def __init__(self) -> None:
+        self._lock = Lock()
+        self._active_items: dict[str, ItemTracker] = {}
+        self._status_counts: dict[OperationStatus, int] = defaultdict(int)
+        self._issue_counts: dict[OperationStatus, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+
+    def add_issue(self, item_id: str, issue: str) -> None:
+        """Add an issue to an item, creating tracker if needed."""
+        with self._lock:
+            if item_id not in self._active_items:
+                self._active_items[item_id] = ItemTracker(item_id)
+            self._active_items[item_id].add_issue(issue)
+
+    def finalize_item(self, item_id: str | list[str], status: OperationStatus) -> None:
+        """Finalize an item with its final status.
+
+        Args:
+            item_id: The item's identifier.
+            status: Final status (success, failure, unchanged).
+        """
+        with self._lock:
+            if isinstance(item_id, list):
+                for iid in item_id:
+                    self._finalize_item_unlocked(iid, status)
+            else:
+                self._finalize_item_unlocked(item_id, status)
+
+    def _finalize_item_unlocked(self, item_id: str, status: OperationStatus) -> None:
+        """Internal method to finalize an item without acquiring the lock."""
+        tracker = self._active_items.pop(item_id, None)
+        self._status_counts[status] += 1
+        if tracker is not None:
+            for issue in tracker.issues:
+                self._issue_counts[status][issue] += 1
+
+    def get_status_counts(self) -> dict[OperationStatus, int]:
+        """Get counts per final status."""
+        with self._lock:
+            return dict(self._status_counts)
+
+    def get_issue_counts(self, status: OperationStatus) -> dict[str, int]:
+        """Get issue counts, optionally filtered by status."""
+        with self._lock:
+            return dict(self._issue_counts.get(status, {}))
+
+
+class DataLogger(ABC):
+    """Abstract base class for data loggers that track operations and log entries."""
+
+    tracker: OperationTracker
+
+    @abstractmethod
+    def log(self, entry: LogEntry | Sequence[LogEntry]) -> None:
+        """Log a detailed entry."""
+        raise NotImplementedError()
+
+
+class NoOpLogger(DataLogger):
+    """A no-op logger that discards all log entries and does no tracking."""
+
+    def __init__(self) -> None:
+        self.tracker = NoOpTracker()
+
+    def log(self, entry: LogEntry | Sequence[LogEntry]) -> None:
+        """Discard the log entry (no-op)."""
+        pass
+
+
+class FileDataLogger(DataLogger):
+    """Composes aggregation tracking with detailed file logging."""
+
+    def __init__(self, writer: NDJsonWriter) -> None:
+        self.tracker = MemoryOperationTracker()
+        self._writer = writer
+
+    def log(self, entry: LogEntry | Sequence[LogEntry]) -> None:
+        """Log a detailed entry to the file."""
+        entries = entry if isinstance(entry, Sequence) else [entry]
+        self._writer.write_chunks([e.model_dump(by_alias=True) for e in entries])

cognite_toolkit/_cdf_tk/tk_warnings/__init__.py

@@ -12,6 +12,7 @@ from .fileread import (
     EnvironmentVariableMissingWarning,
     FileExistsWarning,
     FileReadWarning,
+    FunctionRequirementsValidationWarning,
     MissingFileWarning,
     MissingReferencedWarning,
     MissingRequiredParameterWarning,
@@ -51,6 +52,7 @@ __all__ = [
     "EnvironmentVariableMissingWarning",
     "FileExistsWarning",
     "FileReadWarning",
+    "FunctionRequirementsValidationWarning",
     "GeneralWarning",
     "HTTPWarning",
     "HighSeverityWarning",

cognite_toolkit/_cdf_tk/tk_warnings/fileread.py

@@ -270,6 +270,26 @@ class StreamlitRequirementsWarning(FileReadWarning):
         return f"Missing dependencies in requirements.txt: {', '.join(self.dependencies)}"
 
 
+@dataclass(frozen=True)
+class FunctionRequirementsValidationWarning(FileReadWarning):
+    severity: ClassVar[SeverityLevel] = SeverityLevel.HIGH
+    function_external_id: str
+    error_details: str
+    is_credential_error: bool
+
+    def get_message(self) -> str:
+        message = (
+            f"Function [bold]{self.function_external_id}[/bold] requirements.txt validation failed. "
+            f"Packages could not be resolved: {self.error_details}"
+        )
+        if self.is_credential_error:
+            message += (
+                f"\n{HINT_LEAD_TEXT}This appears to be a credential/authentication issue. "
+                "Check if the Personal Access Token (PAT) or credentials in indexUrl are valid and not expired."
+            )
+        return message
+
+
 @dataclass(frozen=True)
 class ResourceFormatWarning(FileReadWarning):
     severity: ClassVar[SeverityLevel] = SeverityLevel.HIGH

cognite_toolkit/_cdf_tk/utils/__init__.py

@@ -19,11 +19,19 @@ from .hashing import (
     calculate_hash,
     calculate_secure_hash,
 )
-from .modules import find_directory_with_subdirectories, iterate_modules, module_from_path, resource_folder_from_path
+from .modules import (
+    find_directory_with_subdirectories,
+    iterate_modules,
+    module_from_path,
+    module_path_display_name,
+    resource_folder_from_path,
+)
+from .pip_validator import PipValidationResult, validate_requirements_with_pip
 from .sentry_utils import sentry_exception_filter
 
 __all__ = [
     "GraphQLParser",
+    "PipValidationResult",
     "YAMLComment",
     "YAMLWithComments",
     "calculate_directory_hash",
@@ -37,6 +45,7 @@ __all__ = [
     "iterate_modules",
     "load_yaml_inject_variables",
     "module_from_path",
+    "module_path_display_name",
     "quote_int_value_by_key_in_yaml",
     "read_yaml_content",
     "read_yaml_file",
@@ -50,4 +59,5 @@ __all__ = [
     "stringify_value_by_key_in_yaml",
     "tmp_build_directory",
     "to_diff",
+    "validate_requirements_with_pip",
 ]

cognite_toolkit/_cdf_tk/utils/cdf.py

@@ -21,7 +21,7 @@ from filelock import BaseFileLock, FileLock, Timeout
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawTable
 from cognite_toolkit._cdf_tk.constants import ENV_VAR_PATTERN, MAX_ROW_ITERATION_RUN_QUERY, MAX_RUN_QUERY_FREQUENCY_MIN
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitError,

cognite_toolkit/_cdf_tk/utils/interactive_select.py

@@ -21,11 +21,11 @@ from questionary import Choice
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.canvas import Canvas
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.charts import Chart, ChartList, Visibility
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.migration import ResourceViewMapping
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
-from cognite_toolkit._cdf_tk.client.data_classes.three_d import ThreeDModelResponse
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.canvas import Canvas
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.charts import Chart, ChartList, Visibility
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawTable
+from cognite_toolkit._cdf_tk.client.resource_classes.three_d import ThreeDModelResponse
 from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError, ToolkitValueError
 
 from . import humanize_collection
@@ -852,7 +852,9 @@ class ThreeDInteractiveSelect:
             ],
         ).ask()
 
-        models = self.client.tool.three_d.models.list(published=published, include_revision_info=True, limit=None)
+        models = self.client.tool.three_d.models_classic.list(
+            published=published, include_revision_info=True, limit=None
+        )
         if model_type == "classic":
             models = [model for model in models if model.space is None]
         else:

cognite_toolkit/_cdf_tk/utils/modules.py

@@ -12,6 +12,7 @@ from cognite_toolkit._cdf_tk.constants import (
 def iterate_modules(root_dir: Path) -> Iterator[tuple[Path, list[Path]]]:
     """Iterate over all modules in the project and yield the module directory and all files in the module.
 
+
     Args:
         root_dir (Path): The root directory of the project
 
@@ -110,6 +111,12 @@ def is_module_path(path: Path) -> bool:
     return any(sub_folder.name in CRUDS_BY_FOLDER_NAME for sub_folder in path.iterdir() if sub_folder.is_dir())
 
 
+def module_path_display_name(org_path: Path, module_path: Path) -> str:
+    """Returns the module directory path relative to the organization path as a posix string."""
+    rel = module_path.relative_to(org_path)
+    return rel.as_posix()
+
+
 def find_directory_with_subdirectories(
     directory_name: str | None, root_directory: Path
 ) -> tuple[Path | None, list[str]]:

cognite_toolkit/_cdf_tk/utils/pip_validator.py

@@ -0,0 +1,96 @@
+"""Utilities for validating Python package requirements."""
+
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+# Maximum number of error lines to include in warnings
+_MAX_ERROR_LINES = 3
+
+
+@dataclass
+class PipValidationResult:
+    """Result from validating a requirements.txt file."""
+
+    error_message: str | None = None
+
+    @property
+    def success(self) -> bool:
+        """Validation succeeded if there's no error message."""
+        return self.error_message is None
+
+    @property
+    def is_credential_error(self) -> bool:
+        """Check if the error appears to be related to authentication/credentials.
+
+        Only checks for explicit HTTP authentication errors to avoid false positives
+        from legitimate package not found errors.
+        """
+        if not self.error_message:
+            return False
+        credential_indicators = [
+            "401",
+            "403",
+            "Unauthorized",
+            "Forbidden",
+            "Authentication",
+        ]
+        return any(indicator in self.error_message for indicator in credential_indicators)
+
+
+def validate_requirements_with_pip(
+    requirements_txt_path: Path,
+    index_url: str | None = None,
+    extra_index_urls: list[str] | None = None,
+    timeout: int = 10,
+) -> PipValidationResult:
+    """Validate that requirements.txt can be resolved using pip install --dry-run.
+
+    This simulates package installation without actually installing anything.
+    It validates that:
+    - All packages exist
+    - All specified versions are available
+    - Credentials for private repositories are valid
+    - Index URLs are accessible
+
+    Args:
+        requirements_txt_path: Path to the requirements.txt file
+        index_url: Optional custom package index URL (replaces PyPI)
+        extra_index_urls: Optional additional package index URLs
+        timeout: Timeout in seconds for the pip command
+
+    Returns:
+        PipValidationResult with success status and error details if failed
+
+    """
+    if not requirements_txt_path.exists():
+        return PipValidationResult(error_message=f"Requirements file not found: {requirements_txt_path}")
+
+    # Build pip command with custom index URLs
+    args = [
+        sys.executable,
+        "-m",
+        "pip",
+        "install",
+        "--dry-run",
+        "--ignore-installed",
+        "--no-deps",
+        "--disable-pip-version-check",
+        "-r",
+        str(requirements_txt_path),
+        *(["--index-url", index_url] if index_url else []),
+        *([arg for url in (extra_index_urls or []) for arg in ["--extra-index-url", url]]),
+    ]
+
+    try:
+        result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=False)
+        if result.returncode == 0:
+            return PipValidationResult()
+        return PipValidationResult(
+            error_message=f"pip validation failed with exit code {result.returncode}:\n{result.stderr}",
+        )
+    except subprocess.TimeoutExpired:
+        return PipValidationResult(error_message=f"pip validation timed out after {timeout} seconds")
+    except (OSError, RuntimeError) as e:
+        return PipValidationResult(error_message=f"Error running pip validation: {e!s}")