datamint 2.3.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

datamint/entities/cache_manager.py

@@ -105,7 +105,11 @@ class CacheManager(Generic[T]):
         Returns:
             Path to the data file
         """
-        return self._get_entity_cache_dir(entity_id) / f"{data_key}.pkl"
+
+        datapath = self._get_entity_cache_dir(entity_id) / f"{data_key}"
+        if datapath.with_suffix('.pkl').exists():
+            return datapath.with_suffix('.pkl')
+        return datapath
 
     def _compute_version_hash(self, version_info: dict[str, Any]) -> str:
         """Compute a hash from version information.
@@ -120,13 +124,13 @@ class CacheManager(Generic[T]):
         sorted_info = json.dumps(version_info, sort_keys=True)
         return hashlib.sha256(sorted_info.encode()).hexdigest()
 
-    def get(
+    def _get_validated_metadata(
         self,
         entity_id: str,
         data_key: str,
         version_info: dict[str, Any] | None = None
-    ) -> T | None:
-        """Retrieve cached data for an entity.
+    ) -> tuple['CacheManager.ItemMetadata', Path] | tuple[None, None]:
+        """Get and validate cached metadata for an entity.
 
         Args:
             entity_id: Unique identifier for the entity
@@ -134,42 +138,152 @@ class CacheManager(Generic[T]):
             version_info: Optional version information from server to validate cache
 
         Returns:
-            Cached data if valid, None if cache miss or invalid
+            Tuple of (metadata, data_path) if valid, (None, None) if cache miss or invalid
         """
         metadata_path = self._get_metadata_path(entity_id)
-        data_path = self._get_data_path(entity_id, data_key)
 
-        # Check if cache exists
-        if not metadata_path.exists() or not data_path.exists():
-            _LOGGER.debug(f"Cache miss for {entity_id}/{data_key}")
-            return None
+        if not metadata_path.exists():
+            _LOGGER.debug(f"Cache miss for {entity_id}/{data_key} - no metadata")
+            return None, None
 
         try:
-            # Load or create metadata
+            # Read metadata first to get the actual data path (could be external)
             with open(metadata_path, 'r') as f:
                 jsondata = f.read()
             cached_metadata = CacheManager.ItemMetadata.model_validate_json(jsondata)
+            
+            # Use the data_path from metadata (supports external file locations)
+            data_path = Path(cached_metadata.data_path)
+            
+            # Check if the actual data file exists
+            if not data_path.exists():
+                _LOGGER.debug(f"Cache miss for {entity_id}/{data_key} - data file not found at {data_path}")
+                return None, None
 
             # Validate version if provided
             if version_info is not None:
                 server_version = self._compute_version_hash(version_info)
-
                 if server_version != cached_metadata.version_hash:
                     _LOGGER.debug(
                         f"Cache version mismatch for {entity_id}/{data_key}. "
                         f"Server: {server_version}, Cached: {cached_metadata.version_hash}"
                     )
-                    return None
+                    return None, None
 
-            data = self._load_data(cached_metadata)
+            return cached_metadata, data_path
+        except Exception as e:
+            _LOGGER.warning(f"Error reading cache metadata for {entity_id}/{data_key}: {e}")
+            return None, None
 
+    def get(
+        self,
+        entity_id: str,
+        data_key: str,
+        version_info: dict[str, Any] | None = None
+    ) -> T | None:
+        """Retrieve cached data for an entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Key identifying the type of data
+            version_info: Optional version information from server to validate cache
+
+        Returns:
+            Cached data if valid, None if cache miss or invalid
+        """
+        cached_metadata, data_path = self._get_validated_metadata(entity_id, data_key, version_info)
+        
+        if cached_metadata is None:
+            return None
+
+        try:
+            data = self._load_data(cached_metadata)
             _LOGGER.debug(f"Cache hit for {entity_id}/{data_key}")
             return data
-
         except Exception as e:
             _LOGGER.warning(f"Error reading cache for {entity_id}/{data_key}: {e}")
             return None
 
+    def get_path(
+        self,
+        entity_id: str,
+        data_key: str,
+        version_info: dict[str, Any] | None = None
+    ) -> Path | None:
+        """Get the path to cached data for an entity if valid.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Key identifying the type of data
+            version_info: Optional version information from server to validate cache
+
+        Returns:
+            Path to cached data if valid, None if cache miss or invalid
+        """
+        cached_metadata, data_path = self._get_validated_metadata(entity_id, data_key, version_info)
+        return data_path
+
+    def get_expected_path(self, entity_id: str, data_key: str) -> Path:
+        """Get the expected cache path for an entity (even if not yet cached).
+
+        This is useful for downloading directly to the cache location.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Key identifying the type of data
+
+        Returns:
+            Path where data will be cached
+        """
+        return self._get_data_path(entity_id, data_key)
+
+    def register_file_location(
+        self,
+        entity_id: str,
+        data_key: str,
+        file_path: str | Path,
+        version_info: dict[str, Any] | None = None,
+        mimetype: str = 'application/octet-stream'
+    ) -> None:
+        """Register an external file location in cache metadata without copying data.
+
+        This allows tracking a file stored at an arbitrary location (e.g., user's save_path)
+        while keeping version metadata in the cache directory.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Key identifying the type of data
+            file_path: Path to the external file to register
+            version_info: Optional version information from server
+            mimetype: MIME type of the file data
+        """
+        metadata_path = self._get_metadata_path(entity_id)
+        file_path = Path(file_path).resolve().absolute()
+
+        if not file_path.exists():
+            raise FileNotFoundError(f"Cannot register non-existent file: {file_path}")
+
+        try:
+            metadata = CacheManager.ItemMetadata(
+                cached_at=datetime.now(),
+                data_path=str(file_path),
+                data_type='bytes',
+                mimetype=mimetype,
+                entity_id=entity_id
+            )
+
+            if version_info is not None:
+                metadata.version_hash = self._compute_version_hash(version_info)
+                metadata.version_info = version_info
+
+            with open(metadata_path, 'w') as f:
+                f.write(metadata.model_dump_json(indent=2))
+
+            _LOGGER.debug(f"Registered external file for {entity_id}/{data_key}: {file_path}")
+
+        except Exception as e:
+            _LOGGER.warning(f"Error registering file location for {entity_id}/{data_key}: {e}")
+
     def set(
         self,
         entity_id: str,

datamint/entities/datasetinfo.py

@@ -1,8 +1,8 @@
 """Dataset entity module for DataMint API."""
 
-from datetime import datetime
 import logging
 from typing import TYPE_CHECKING, Sequence
+from pydantic import PrivateAttr
 
 from .base_entity import BaseEntity, MISSING_FIELD
 
@@ -10,6 +10,7 @@ if TYPE_CHECKING:
     from datamint.api.client import Api
     from .resource import Resource
     from .project import Project
+    from datamint.api.endpoints.datasetsinfo_api import DatasetsInfoApi
 
 logger = logging.getLogger(__name__)
 
@@ -31,92 +32,86 @@ class DatasetInfo(BaseEntity):
     total_resource: int
     resource_ids: list[str]
 
+    _api: 'DatasetsInfoApi' = PrivateAttr()
+
     def __init__(self, **data):
         """Initialize the dataset info entity."""
         super().__init__(**data)
-        self._manager: EntityManager['DatasetInfo'] = EntityManager(self)
         
         # Cache for lazy-loaded data
-        self._resources_cache: Sequence['Resource'] | None = None
-        self._projects_cache: Sequence['Project'] | None = None
+        # self._resources_cache: Sequence['Resource'] | None = None
+        # self._projects_cache: Sequence['Project'] | None = None
     
-    def _inject_api(self, api: 'Api') -> None:
-        """Inject API client into this dataset (called automatically by Api class)."""
-        self._manager.set_api(api)
-
-    def get_resources(
-        self,
-        refresh: bool = False,
-        limit: int | None = None
-    ) -> Sequence['Resource']:
-        """Get all resources in this dataset.
+    # def get_resources(
+    #     self,
+    #     refresh: bool = False,
+    #     limit: int | None = None
+    # ) -> Sequence['Resource']:
+    #     """Get all resources in this dataset.
         
-        Results are cached after the first call unless refresh=True.
+    #     Results are cached after the first call unless refresh=True.
         
-        Args:
-            api: Optional API client. Uses the one from set_api() if not provided.
-            refresh: If True, bypass cache and fetch fresh data
+    #     Args:
+    #         api: Optional API client. Uses the one from set_api() if not provided.
+    #         refresh: If True, bypass cache and fetch fresh data
             
-        Returns:
-            List of Resource instances in this dataset
+    #     Returns:
+    #         List of Resource instances in this dataset
             
-        Raises:
-            RuntimeError: If no API client is available
+    #     Raises:
+    #         RuntimeError: If no API client is available
             
-        Example:
-            >>> dataset = api._datasetsinfo.get_by_id("dataset-id")
-            >>> dataset.set_api(api)
-            >>> resources = dataset.get_resources()
-        """
-        if refresh or self._resources_cache is None:
-            api_client = self._manager._ensure_api(api)
+    #     Example:
+    #         >>> dataset = api._datasetsinfo.get_by_id("dataset-id")
+    #         >>> dataset.set_api(api)
+    #         >>> resources = dataset.get_resources()
+    #     """
+    #     if refresh or self._resources_cache is None:
+    #         # Fetch resources by their IDs
+    #         resources = []
+    #         for resource_id in self.resource_ids:
+    #             try:
+    #                 resource = self._api.get.get_by_id(resource_id)
+    #                 resource.set_api(self._api)
+    #                 resources.append(resource)
+    #             except Exception as e:
+    #                 logger.warning(f"Failed to fetch resource {resource_id}: {e}")
             
-            # Fetch resources by their IDs
-            resources = []
-            for resource_id in self.resource_ids:
-                try:
-                    resource = api_client.resources.get_by_id(resource_id)
-                    resource.set_api(api_client)
-                    resources.append(resource)
-                except Exception as e:
-                    logger.warning(f"Failed to fetch resource {resource_id}: {e}")
-            
-            self._resources_cache = resources
+    #         self._resources_cache = resources
         
-        return self._resources_cache
+    #     return self._resources_cache
 
-    def get_projects(
-        self,
-        api: 'Api | None' = None,
-        refresh: bool = False
-    ) -> Sequence['Project']:
-        """Get all projects associated with this dataset.
+    # def get_projects(
+    #     self,
+    #     api: 'Api | None' = None,
+    #     refresh: bool = False
+    # ) -> Sequence['Project']:
+    #     """Get all projects associated with this dataset.
         
-        Results are cached after the first call unless refresh=True.
+    #     Results are cached after the first call unless refresh=True.
         
-        Args:
-            refresh: If True, bypass cache and fetch fresh data
+    #     Args:
+    #         refresh: If True, bypass cache and fetch fresh data
             
-        Returns:
-            List of Project instances
+    #     Returns:
+    #         List of Project instances
             
-        Raises:
-            RuntimeError: If no API client is available
+    #     Raises:
+    #         RuntimeError: If no API client is available
             
-        Example:
-            >>> dataset = api.datasetsinfo.get_by_id("dataset-id")
-            >>> projects = dataset.get_projects()
-        """
-        if refresh or self._projects_cache is None:
-            api_client = self._manager.api
+    #     Example:
+    #         >>> dataset = api.datasetsinfo.get_by_id("dataset-id")
+    #         >>> projects = dataset.get_projects()
+    #     """
+    #     if refresh or self._projects_cache is None:
             
-            # Get all projects and filter by dataset_id
-            all_projects = api_client.projects.get_all()
-            projects = [p for p in all_projects if p.dataset_id == self.id]
+    #         # Get all projects and filter by dataset_id
+    #         all_projects = api_client.projects.get_all()
+    #         projects = [p for p in all_projects if p.dataset_id == self.id]
             
-            self._projects_cache = projects
+    #         self._projects_cache = projects
         
-        return self._projects_cache
+    #     return self._projects_cache
     
     def invalidate_cache(self) -> None:
         """Invalidate all cached relationship data.

datamint/entities/deployjob.py

@@ -0,0 +1,18 @@
+from datamint.entities.base_entity import BaseEntity
+
+
+class DeployJob(BaseEntity):
+    id: str
+    status: str
+    model_name: str
+    model_version: int | None = None
+    model_alias: str | None = None
+    image_name: str | None = None
+    image_tag: str | None = None
+    error_message: str | None = None
+    progress_percentage: int = 0
+    current_step: str | None = None
+    with_gpu: bool = False
+    recent_logs: list[str] | None = None
+    started_at: str | None = None
+    completed_at: str | None = None

datamint/entities/project.py

@@ -75,6 +75,45 @@ class Project(BaseEntity):
         """
         return self._api.get_project_resources(self.id)
 
+    def download_resources_datas(self, progress_bar: bool = True) -> None:
+        """Downloads all project resources in parallel for faster subsequent access.
+
+        This method downloads and caches all resource file data concurrently,
+        skipping resources that are already cached. This dramatically improves
+        performance when working with large projects.
+
+        Args:
+            progress_bar: Whether to show a progress bar. Default is True.
+
+        Example:
+            >>> proj = api.projects.get_by_name("My Project")
+            >>> proj.download_resources()  # Cache all resources in parallel
+            >>> # Now fetch_file_data() will be instantaneous for cached resources
+            >>> for res in proj.fetch_resources():
+            ...     data = res.fetch_file_data(use_cache=True)
+        """
+        return self.cache_resources(progress_bar=progress_bar)
+
+    def cache_resources(self, progress_bar: bool = True) -> None:
+        """Cache all project resources in parallel for faster subsequent access.
+
+        This method downloads and caches all resource file data concurrently,
+        skipping resources that are already cached. This dramatically improves
+        performance when working with large projects.
+
+        Args:
+            progress_bar: Whether to show a progress bar. Default is True.
+
+        Example:
+            >>> proj = api.projects.get_by_name("My Project")
+            >>> proj.cache_resources()  # Cache all resources in parallel
+            >>> # Now fetch_file_data() will be instantaneous for cached resources
+            >>> for res in proj.fetch_resources():
+            ...     data = res.fetch_file_data(use_cache=True)
+        """
+        resources = self.fetch_resources()
+        self._api.resources_api.cache_resources(resources, progress_bar=progress_bar)
+
     def set_work_status(self, resource: 'Resource', status: Literal['opened', 'annotated', 'closed']) -> None:
         """Set the status of a resource.